diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 3373e93fc..5728cbb30 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -41,8 +41,8 @@ def check_up if ScheduleSnapshot.where("status = 'in_progress'").count > 0 redirect_to '/maintenance.html', status: 503 end - # Stop people from making changes if we are publishing - if PublicationStatus.where("status = 'inprogress'").count > 0 + # Stop people from making changes if we are running any long job + if JobStatus.where("status = 'inprogress'").count > 0 redirect_to '/maintenance.html', status: 503 end end diff --git a/app/controllers/concerns/resource_methods.rb b/app/controllers/concerns/resource_methods.rb index 1b244214b..d0495464d 100644 --- a/app/controllers/concerns/resource_methods.rb +++ b/app/controllers/concerns/resource_methods.rb @@ -230,11 +230,17 @@ def collection @per_page, @current_page, @filters = collection_params + q = if select_fields select_fields else policy_scope(base, policy_scope_class: policy_scope_class) end + + if default_scope(query: q) + q = default_scope(query: q) + end + q = q.includes(includes) .references(references) .eager_load(eager_load) @@ -243,7 +249,7 @@ def collection .where(collection_where) # anpther where? - q = q.distinct if join_tables && !join_tables.empty? + q = q.distinct if (join_tables && !join_tables.empty?) || make_distinct? q = q.order(order_string) @@ -255,8 +261,11 @@ def collection # TODO we need the size without the query if paginated - @full_collection_total = policy_scope(base, policy_scope_class: policy_scope_class) - .where(exclude_deleted_clause) + fq = policy_scope(base, policy_scope_class: policy_scope_class) + if default_scope(query: fq) + fq = default_scope(query: fq) + end + @full_collection_total = fq.where(exclude_deleted_clause) .includes(includes) .references(references) .eager_load(eager_load) @@ -527,6 +536,10 @@ def select_fields nil end + def default_scope(query: nil) + nil + end + def model_name "#{model_class}" end @@ -749,6 +762,10 @@ def array_col?(col_name:) false end + def make_distinct? + false + end + def permitted_params() _permitted_params(model: nil) end diff --git a/app/controllers/person_sync_data_controller.rb b/app/controllers/person_sync_data_controller.rb new file mode 100644 index 000000000..de35f8ee1 --- /dev/null +++ b/app/controllers/person_sync_data_controller.rb @@ -0,0 +1,101 @@ +class PersonSyncDataController < ResourceController + SERIALIZER_CLASS = 'PersonSyncDatumSerializer'.freeze + POLICY_CLASS = 'PersonSyncDatumPolicy'.freeze + POLICY_SCOPE_CLASS = 'PersonSyncDatumPolicy::Scope'.freeze + DEFAULT_SORTBY = 'name_sort_by' + DEFAULT_ORDER = 'asc'.freeze + + # + # + # + def match + model_class.transaction do + authorize model_class, policy_class: policy_class + + reg_id = params[:reg_id] if params[:reg_id] + person_id = params[:person_id] if params[:person_id] + + # one of 'assisted' or 'manual' + reg_match = params[:reg_match] if params[:reg_match] + + raise "Type of match should be 'assisted' or 'manual' you gave '#{reg_match}'" unless ['assisted', 'manual'].include? reg_match + raise "No reg id, person id, or match type specified" unless reg_id && person_id && reg_match + + # Get the reg sync data + datum = RegistrationSyncDatum.find_by reg_id: reg_id + + # Get the person + person = Person.find person_id + + # Update the person with the reg data + IdentityService.clear_person_reg_info(person: person); + IdentityService.update_reg_info(person: person, details: datum.raw_info, reg_match: reg_match) + + render status: :ok, + json: { message: "Matched" }.to_json, + content_type: 'application/json' + end + end + + # + # Method to dismiss a match + # POST request, parameters are reg_id and person_id + # + def dismiss_match + model_class.transaction do + authorize model_class, policy_class: policy_class + + reg_id = params[:reg_id] if params[:reg_id] + person_id = params[:person_id] if params[:person_id] + + raise "No reg id or person id specified" unless reg_id && person_id + + existing = DismissedRegSyncMatch.find_by reg_id: reg_id, person_id: person_id + + if !existing + DismissedRegSyncMatch.create!({ + reg_id: reg_id, + person_id: person_id + }) + end + + render status: :ok, + json: { message: "Dismissed Match" }.to_json, + content_type: 'application/json' + end + end + + # by default get the data that is not already mapped to a person + def default_scope(query: nil) + return nil unless query + + # People that have a potential mapping and not already mapped + query.joins(:registration_sync_data) + .where('people.reg_id is null') + .where('registration_sync_data.reg_id in (select reg_id from registration_map_counts)') + end + + def select_fields + PersonSyncDatum.select( + ::PersonSyncDatum.arel_table[Arel.star], + 'name_sort_by' + ) + end + + def serializer_includes + [ + :registration_sync_data + ] + end + + def make_distinct? + true + end + + def includes + [ + :email_addresses, + :registration_sync_data + ] + end +end diff --git a/app/controllers/registration_sync_data_controller.rb b/app/controllers/registration_sync_data_controller.rb new file mode 100644 index 000000000..a9f16e907 --- /dev/null +++ b/app/controllers/registration_sync_data_controller.rb @@ -0,0 +1,63 @@ +class RegistrationSyncDataController < ResourceController + SERIALIZER_CLASS = 'RegistrationSyncDatumSerializer'.freeze + POLICY_CLASS = 'RegistrationSyncDatumPolicy'.freeze + POLICY_SCOPE_CLASS = 'RegistrationSyncDatumPolicy::Scope'.freeze + DEFAULT_SORTBY = 'registration_sync_data.name' + DEFAULT_ORDER = 'asc'.freeze + + def sync_statistics + authorize current_person, policy_class: policy_class + status = RegistrationSyncStatus.order('created_at desc').first + + result = status ? status.result : {} + + render status: :ok, json: result.to_json, content_type: 'application/json' + end + + def synchronize + authorize current_person, policy_class: policy_class + + status = RegistrationSyncStatus.order('created_at desc').first + status = RegistrationSyncStatus.new if status == nil + if status.status != 'inprogress' + status.status = 'inprogress' + status.save! + + RegistrationSyncWorker.perform_async + end + + render status: :ok, json: {}.to_json, content_type: 'application/json' + end + + def people + authorize model_class, policy_class: policy_class + datum = RegistrationSyncDatum.find params[:registration_sync_datum_id] + + people = datum.people + options = { + params: { + domain: "#{request.base_url}", + current_person: current_person + } + } + + # return the list of people associated with this datum + render json: PersonSerializer.new(people,options).serializable_hash(), + content_type: 'application/json' + end + + def serializer_includes + [ + :matched_person + ] + end + + # # by default get the data that is not already mapped to a person + # def default_scope(query: nil) + # return nil unless query + + # # People that have a potential mapping and not already mapped + # query.where('reg_id not in (select reg_id from people where reg_id is not null)') + # .where('reg_id in (select reg_id from registration_map_counts)') + # end +end diff --git a/app/javascript/administration/admin_registrations.vue b/app/javascript/administration/admin_registrations.vue new file mode 100644 index 000000000..e0a165ca2 --- /dev/null +++ b/app/javascript/administration/admin_registrations.vue @@ -0,0 +1,41 @@ + + + + + diff --git a/app/javascript/administration/playground_component.vue b/app/javascript/administration/playground_component.vue index 0a09e90cd..2651c76e3 100644 --- a/app/javascript/administration/playground_component.vue +++ b/app/javascript/administration/playground_component.vue @@ -1,23 +1,18 @@ diff --git a/app/javascript/navbar/side-navbar.vue b/app/javascript/navbar/side-navbar.vue index d3ad47ad5..0af60cdf4 100644 --- a/app/javascript/navbar/side-navbar.vue +++ b/app/javascript/navbar/side-navbar.vue @@ -2,7 +2,7 @@
Dashboard - Venues + People
diff --git a/app/javascript/people/people_admin_tab.vue b/app/javascript/people/people_admin_tab.vue index f297aaab7..5f84b33e8 100644 --- a/app/javascript/people/people_admin_tab.vue +++ b/app/javascript/people/people_admin_tab.vue @@ -1,64 +1,137 @@ - + diff --git a/app/javascript/people/person-edit-reg-number.vue b/app/javascript/people/person-edit-reg-number.vue new file mode 100644 index 000000000..97eb43505 --- /dev/null +++ b/app/javascript/people/person-edit-reg-number.vue @@ -0,0 +1,130 @@ + + + + + diff --git a/app/javascript/profile/dl_person.vue b/app/javascript/profile/dl_person.vue index b273fb920..fa7ea454a 100644 --- a/app/javascript/profile/dl_person.vue +++ b/app/javascript/profile/dl_person.vue @@ -9,10 +9,11 @@
Restricted - Yes - No - Not Specified + {{ yes(field) }} + {{ no(field) }} + {{ notSpecified(field) }} {{selected[field]}} +
@@ -29,6 +30,12 @@ export default { props: { fields: { default: [] + }, + overrides: { + default: () => {} + }, + nullText: { + default: "Not Specified" } }, mixins: [ @@ -38,5 +45,19 @@ export default { model, PROFILE_FIELD_LABELS }), + methods: { + getOverride(key, defaultText, field) { + return this.overrides?.[key]?.[field] ?? defaultText; + }, + notSpecified(field) { + return this.getOverride('null', this.nullText, field); + }, + yes(field) { + return this.getOverride('true', 'Yes', field); + }, + no(field) { + return this.getOverride('false', 'No', field); + } + } } diff --git a/app/javascript/profile/person_details.vue b/app/javascript/profile/person_details.vue index 0135090ac..a1fe2f829 100644 --- a/app/javascript/profile/person_details.vue +++ b/app/javascript/profile/person_details.vue @@ -4,8 +4,9 @@
Identity
- - + + +
diff --git a/app/javascript/registrations/display_sync_data.vue b/app/javascript/registrations/display_sync_data.vue new file mode 100644 index 000000000..72273a25a --- /dev/null +++ b/app/javascript/registrations/display_sync_data.vue @@ -0,0 +1,31 @@ + + + + + diff --git a/app/javascript/registrations/person_sync_columns.js b/app/javascript/registrations/person_sync_columns.js new file mode 100644 index 000000000..45d501090 --- /dev/null +++ b/app/javascript/registrations/person_sync_columns.js @@ -0,0 +1,31 @@ + +export const person_sync_columns = [ + { + key: 'published_name', + label: 'Published Name', + type: "text", + sortable: true, + class: 'col-name-field', + stickyColumn: true + }, + { + key: 'name', + label: 'Name', + type: "text", + sortable: true, + class: 'col-name-field' + }, + { + key: 'primary_email', + search_key: 'email_addresses.email', + label: 'Email', + type: "text", + sortable: false + }, + { + key: 'registration_sync_data', + label: 'Possible Match(es)', + // Type should be JSON ... + type: "text" + } +] diff --git a/app/javascript/registrations/person_sync_table.vue b/app/javascript/registrations/person_sync_table.vue new file mode 100644 index 000000000..9a6d16e35 --- /dev/null +++ b/app/javascript/registrations/person_sync_table.vue @@ -0,0 +1,71 @@ + + + diff --git a/app/javascript/registrations/registration_sync_columns.js b/app/javascript/registrations/registration_sync_columns.js new file mode 100644 index 000000000..c3950a473 --- /dev/null +++ b/app/javascript/registrations/registration_sync_columns.js @@ -0,0 +1,36 @@ + +export const registration_sync_columns = [ + { + key: 'name', + label: 'Name', + type: "text" + }, + { + key: 'registration_number', + label: 'Registration Number', + type: "text" + }, + // { + // key: 'reg_id', + // label: 'Registration Id', + // type: "text" + // }, + { + key: 'email', + label: 'Email', + type: "text" + }, + { + key: 'raw_info', + label: 'Raw Info', + // Type should be JSON ... + type: "text" + }, + // TODO: list of people + { + key: 'people', + label: 'Possible Match(es)', + // Type should be JSON ... + type: "text" + } +] diff --git a/app/javascript/registrations/registration_sync_table.vue b/app/javascript/registrations/registration_sync_table.vue new file mode 100644 index 000000000..64d946ac6 --- /dev/null +++ b/app/javascript/registrations/registration_sync_table.vue @@ -0,0 +1,52 @@ + + + diff --git a/app/javascript/schedule/schedulable_sessions.vue b/app/javascript/schedule/schedulable_sessions.vue index e5e0c6f9c..9017d0ab9 100644 --- a/app/javascript/schedule/schedulable_sessions.vue +++ b/app/javascript/schedule/schedulable_sessions.vue @@ -1,27 +1,17 @@ @@ -51,6 +41,9 @@ export default { // Passing the event's data to Vue Cal through the DataTransfer object. e.dataTransfer.setData('event', JSON.stringify(draggable)) e.dataTransfer.setData('cursor-grab-at', e.offsetY) + }, + pillClass(color) { + return `badge badge-pill mr-1 badge-${color} mr-1` } }, mounted() { diff --git a/app/javascript/schedule/schedule_session_search.vue b/app/javascript/schedule/schedule_session_search.vue index 651188f00..f18f25265 100644 --- a/app/javascript/schedule/schedule_session_search.vue +++ b/app/javascript/schedule/schedule_session_search.vue @@ -3,26 +3,37 @@
- +
- +
+
+ + + + + +
+
+ + + + + +
+
- +
@@ -46,6 +57,7 @@ import ModelSelect from '../components/model_select'; import ModelTags from '../components/model_tags'; import searchStateMixin from '../store/search_state.mixin' +import { tagsMixin } from '@/store/tags.mixin'; const SAVED_SEARCH_STATE = "SCHEDULABLE SESSION SELECT STATE"; @@ -56,7 +68,8 @@ export default { ModelTags }, mixins: [ - searchStateMixin + searchStateMixin, + tagsMixin ], props: { columns: Array @@ -67,7 +80,9 @@ export default { area_id: null, tags: null, match: 'any', - schedFilter: 'all' + schedFilter: 'all', + tag: null, + label: null } }, watch: { @@ -101,10 +116,15 @@ export default { ) } - if (this.tags && (this.tags.length > 0)) { - let vals = this.tags.map(obj => (obj.label)) + if (this.tag) { + queries["queries"].push( + ["tags_list_table.tags_array","is",this.tag], + ) + } + + if (this.label) { queries["queries"].push( - ["tags.name","in",vals], + ["labels_list_table.labels_array", "is", this.label], ) } @@ -151,7 +171,8 @@ export default { setting: { title_desc: this.title_desc, area_id: this.area_id, - tags: this.tags, + tag: this.tag, + label: this.label, match: this.match, schedFilter: this.schedFilter } @@ -163,7 +184,8 @@ export default { if (saved) { this.title_desc = saved.title_desc this.area_id = saved.area_id - this.tags = saved.tags + this.tag = saved.tag + this.label = saved.label this.match = saved.match this.schedFilter = saved.schedFilter } diff --git a/app/javascript/schedule/schedule_settings.vue b/app/javascript/schedule/schedule_settings.vue index b103442ec..cfb4b4c68 100644 --- a/app/javascript/schedule/schedule_settings.vue +++ b/app/javascript/schedule/schedule_settings.vue @@ -178,6 +178,7 @@ export default { resetPubs() { this.toastPromise(http.get('/publication_date/reset'), "succesfully reset publication data") }, + // publishdSchedule() { this.toastPromise(http.get('/session/schedule_publish'), "Succesfully requested publish") }, diff --git a/app/javascript/sessions/datetime_picker.vue b/app/javascript/sessions/datetime_picker.vue index 04d153d0d..e3fa94796 100644 --- a/app/javascript/sessions/datetime_picker.vue +++ b/app/javascript/sessions/datetime_picker.vue @@ -58,7 +58,7 @@ export default { let retDate = this.value ? DateTime.fromISO(this.value).setZone(this.conventionTimezone) : DateTime.fromISO(this.conventionStart).setZone(this.conventionTimezone); if (newTime) { console.log('val', newTime, DateTime.fromFormat(newTime, 'HH:mm:ss')) - let time = DateTime.fromFormat(newTime, 'HH:mm:ss', {zone: this.conventionTimezone}).toUTC(); + let time = DateTime.fromFormat(newTime, 'HH:mm:ss', {zone: this.conventionTimezone}); retDate = retDate.set({ hour: time.hour, minute: time.minute, diff --git a/app/javascript/store/model.store.js b/app/javascript/store/model.store.js index b72164390..853cec908 100644 --- a/app/javascript/store/model.store.js +++ b/app/javascript/store/model.store.js @@ -40,6 +40,10 @@ import { venueStore, venueEndpoints} from "@/store/venue.store"; // Page content (html) import { pageContentStore, pageContentEndpoints } from "@/store/page_content.store"; +// Registration Sync Datum/Data +import { registrationSyncDatumStore, registrationSyncDatumEndpoints } from "@/store/registration_sync_datum.store"; +import { personSyncDatumStore, personSyncDatumEndpoints } from "@/store/person_sync_datum.store"; + // mailings import { mailingStore, mailingEndpoints } from './mailing.store'; @@ -105,6 +109,8 @@ const endpoints = { ...roomSetEndpoints, ...venueEndpoints, ...pageContentEndpoints, + ...registrationSyncDatumEndpoints, + ...personSyncDatumEndpoints, ...surveyEndpoints, ...mailingEndpoints, ...sessionEndpoints, @@ -150,6 +156,8 @@ export const store = new Vuex.Store({ ...roomSetStore.selected, ...venueStore.selected, ...pageContentStore.selected, + ...registrationSyncDatumStore.selected, + ...personSyncDatumStore.selected, ...surveyStore.selected, ...mailingStore.selected, ...sessionStore.selected, @@ -200,6 +208,8 @@ export const store = new Vuex.Store({ ...venueStore.getters, ...surveyStore.getters, ...pageContentStore.getters, + ...registrationSyncDatumStore.getters, + ...personSyncDatumStore.getters, ...personSessionStore.getters, ...mailingStore.getters, ...sessionStore.getters, @@ -399,6 +409,8 @@ export const store = new Vuex.Store({ ...roomStore.actions, ...roomSetStore.actions, ...pageContentStore.actions, + ...registrationSyncDatumStore.actions, + ...personSyncDatumStore.actions, ...venueStore.actions, ...mailingStore.actions, ...settingsStore.actions, diff --git a/app/javascript/store/person_sync_datum.mixin.js b/app/javascript/store/person_sync_datum.mixin.js new file mode 100644 index 000000000..70006ec37 --- /dev/null +++ b/app/javascript/store/person_sync_datum.mixin.js @@ -0,0 +1,35 @@ +import { toastMixin } from "@/mixins"; +import { SELECTED } from "./model.store" +import { personModel } from "./person.store" +import { MATCH } from "./person_sync_datum.store"; +import { registrationSyncDatumModel } from "./registration_sync_datum.store" +import { mapActions} from "vuex"; + +export const personSyncDatumMixin = { + mixins: [ + toastMixin + ], + computed: { + selectedPerson() { + return this.$store.getters[SELECTED]({model: personModel}) + }, + selectedRegDatum() { + return this.$store.getters[SELECTED]({model: registrationSyncDatumModel}) + } + }, + methods: { + ...mapActions({ + matchPersonAndReg: MATCH + }), + manualMatch(regId, personId) { + return this.toastPromise(this.matchPersonAndReg({ + regId, + personId, + regMatch: 'manual' + }), "Person successfully linked to Registration") + }, + manualMatchSelected() { + return this.manualMatch(this.selectedRegDatum.reg_id, this.selectedPerson.id); + } + } +} diff --git a/app/javascript/store/person_sync_datum.store.js b/app/javascript/store/person_sync_datum.store.js new file mode 100644 index 000000000..3e9a0d4ad --- /dev/null +++ b/app/javascript/store/person_sync_datum.store.js @@ -0,0 +1,38 @@ +import { http } from '@/http'; +import { FETCH_SELECTED } from './model.store'; +import { personModel } from './person.store'; + +export const personSyncDatumModel = 'person_sync_datum'; + +export const MATCH = "PERSON SYNC MATCH" + +export const personSyncDatumEndpoints = { + [personSyncDatumModel]: 'person_sync_datum' +} + +export const personSyncDatumStore = { + actions: { + [MATCH]({dispatch}, {regId, personId, regMatch}) { + console.log('match action', regId, personId, regMatch) + return new Promise((res, rej) => { + http.post(`${personSyncDatumEndpoints[personSyncDatumModel]}/match`, { + reg_id: regId, + person_id: personId, + reg_match: regMatch + }).then((data) => { + // if it was successful, also then fetch the person. + // todo i think we only wnat to do this sometimes? + dispatch(FETCH_SELECTED, {model: personModel}).then(() => { + res(data); + }) + + }).catch(rej); + }); + } + }, + selected: { + [personSyncDatumModel]: undefined + }, + getters: { + } +} diff --git a/app/javascript/store/registration_sync_datum.store.js b/app/javascript/store/registration_sync_datum.store.js new file mode 100644 index 000000000..76eac90a2 --- /dev/null +++ b/app/javascript/store/registration_sync_datum.store.js @@ -0,0 +1,36 @@ +import { FETCH, SELECT, UNSELECT } from "./model.store"; + +export const registrationSyncDatumModel = 'registration_sync_datum'; +const model = registrationSyncDatumModel; + +export const GET_REG_BY_ID = "GET REG BY ID"; + +export const registrationSyncDatumEndpoints = { + [registrationSyncDatumModel]: 'registration_sync_datum' +} + +export const registrationSyncDatumStore = { + selected: { + [registrationSyncDatumModel]: undefined + }, + getters: { + }, + actions: { + [GET_REG_BY_ID] ({commit, dispatch}, {id}) { + return new Promise((res, rej) => { + dispatch(FETCH, {model, params: { + // trying without the %23 here in the hope that fetch will serialize correctly + filter: `{"op":"all","queries":[["registration_number","is","${id}"]]}` + }}).then((data) => { + const keys = Object.keys(data).filter(key => key !== "_jv") + if(keys.length) { + commit(SELECT, {model, itemOrId: keys[0]}) + } else { + commit(UNSELECT, {model}); + } + res(data); + }).catch(rej); + }) + } + }, +} diff --git a/app/lib/migration_helpers/plano_views.rb b/app/lib/migration_helpers/plano_views.rb index f91cd9f0c..d01cc64b5 100644 --- a/app/lib/migration_helpers/plano_views.rb +++ b/app/lib/migration_helpers/plano_views.rb @@ -25,12 +25,25 @@ def self.create_registration_sync_matches select p.name, null as email, p.id as pid, rsd.reg_id, rsd.id as rid, 'name' as mtype from people p join registration_sync_data rsd - on rsd."name" ilike p.name + on + ( + rsd."name" ilike p.name OR + rsd."preferred_name" ilike p.name OR + rsd."badge_name" ilike p.name + or rsd."name" ilike p.pseudonym + or rsd."preferred_name" ilike p.pseudonym + or rsd."badge_name" ilike p.pseudonym + ) union select null as name, e.email, e.person_id as pid, rsd.reg_id, rsd.id as rid, 'email' as mtype - from email_addresses e + from email_addresses e join registration_sync_data rsd - on rsd."email" ilike e.email + on + ( + rsd."email" ilike e.email or + rsd."alternative_email" ilike e.email + ) + where e.isdefault = true SQL ActiveRecord::Base.connection.execute(query) @@ -40,7 +53,9 @@ def self.create_registration_map_counts query = <<-SQL.squish CREATE OR REPLACE VIEW registration_map_counts AS select rsm.reg_id, rsm.pid, count(rsm.pid) as sub_count - from registration_sync_matches rsm + from registration_sync_matches rsm + where rsm.pid not in (select person_id from dismissed_reg_sync_matches) + and rsm.reg_id not in (select reg_id from dismissed_reg_sync_matches) group by rsm.reg_id, rsm.pid SQL diff --git a/app/models/dismissed_reg_sync_match.rb b/app/models/dismissed_reg_sync_match.rb new file mode 100644 index 000000000..f12b2ac1d --- /dev/null +++ b/app/models/dismissed_reg_sync_match.rb @@ -0,0 +1,21 @@ +# == Schema Information +# +# Table name: dismissed_reg_sync_matches +# +# id :uuid not null, primary key +# lock_version :integer +# created_at :datetime not null +# updated_at :datetime not null +# person_id :uuid not null +# reg_id :string not null +# +# Indexes +# +# idx_person_reg_id (person_id,reg_id) UNIQUE +# index_dismissed_reg_sync_matches_on_person_id (person_id) +# index_dismissed_reg_sync_matches_on_reg_id (reg_id) +# +class DismissedRegSyncMatch < ApplicationRecord + # + belongs_to :person +end diff --git a/app/models/job_status.rb b/app/models/job_status.rb new file mode 100644 index 000000000..59ac4f58d --- /dev/null +++ b/app/models/job_status.rb @@ -0,0 +1,16 @@ +# == Schema Information +# +# Table name: job_statuses +# +# id :uuid not null, primary key +# lock_version :integer default(0) +# result :jsonb +# status :string +# submit_time :datetime +# type :string +# created_at :datetime not null +# updated_at :datetime not null +# +class JobStatus < ApplicationRecord + validates_inclusion_of :status, in: %w[inprogress completed] +end diff --git a/app/models/person.rb b/app/models/person.rb index e6e652d1c..501f9979b 100644 --- a/app/models/person.rb +++ b/app/models/person.rb @@ -66,6 +66,8 @@ # published_name :string # published_name_sort_by :string # reddit :string +# reg_attending_status :string +# reg_match :enum default("none") # registered :boolean default(FALSE), not null # registration_number :string # registration_type :string @@ -114,7 +116,9 @@ class Person < ApplicationRecord # acts_as_taggable acts_as_taggable_on :tags - has_paper_trail versions: { class_name: 'Audit::PersonVersion' }, ignore: [:updated_at, :created_at, :lock_version, :integrations] + has_paper_trail versions: { class_name: 'Audit::PersonVersion' }, + ignore: [:updated_at, :created_at, :lock_version, :integrations], + limit: nil before_destroy :check_if_assigned before_save :check_primary_email @@ -254,6 +258,8 @@ def completed rejected: 'rejected' } + enum reg_match: {none: 'none', automatic: 'automatic', assisted: 'assisted', manual: 'manual', self: 'self'}, _scopes: false + nilify_blanks only: [ :bio, :pseudonym, diff --git a/app/models/person_sync_datum.rb b/app/models/person_sync_datum.rb new file mode 100644 index 000000000..8d2c923d8 --- /dev/null +++ b/app/models/person_sync_datum.rb @@ -0,0 +1,110 @@ +# == Schema Information +# +# Table name: people +# +# id :uuid not null, primary key +# accommodations :string(10000) +# age_at_convention :string +# attendance_type :string(200) +# availability_notes :string +# bio :text +# black_diaspora :string(10000) +# bsky :string +# can_photo :enum default("no") +# can_photo_exceptions :string(10000) +# can_record :enum default("no") +# can_record_exceptions :string(10000) +# can_share :boolean default(FALSE), not null +# can_stream :enum default("no") +# can_stream_exceptions :string(10000) +# comments :text +# con_state :enum default("not_set") +# confirmation_sent_at :datetime +# confirmation_token :string +# confirmed_at :datetime +# current_sign_in_at :datetime +# current_sign_in_ip :inet +# date_reg_synced :datetime +# demographic_categories :string +# do_not_assign_with :string(10000) +# encrypted_password :string default(""), not null +# ethnicity :string(400) +# excluded_demographic_categories :string +# facebook :string +# failed_attempts :integer default(0), not null +# fediverse :string +# flickr :string +# gender :string(400) +# global_diaspora :string +# indigenous :string(10000) +# instagram :string +# integrations :jsonb not null +# is_local :boolean default(FALSE) +# job_title :string +# language :string(5) default("") +# languages_fluent_in :string(10000) +# last_sign_in_at :datetime +# last_sign_in_ip :inet +# linkedin :string +# lock_version :integer default(0) +# locked_at :datetime +# moderation_experience :string(10000) +# name :string default("") +# name_sort_by :string +# name_sort_by_confirmed :boolean default(FALSE) +# needs_accommodations :boolean default(FALSE) +# non_anglophone :string +# non_us_centric_perspectives :string(10000) +# opted_in :boolean default(FALSE), not null +# organization :string +# othered :string(10000) +# othersocialmedia :string +# pronouns :string(400) +# pseudonym :string +# pseudonym_sort_by :string +# pseudonym_sort_by_confirmed :boolean default(FALSE) +# published_name :string +# published_name_sort_by :string +# reddit :string +# reg_attending_status :string +# reg_match :enum default("none") +# registered :boolean default(FALSE), not null +# registration_number :string +# registration_type :string +# remember_created_at :datetime +# reset_password_sent_at :datetime +# reset_password_token :string +# romantic_sexual_orientation :string +# sign_in_count :integer default(0), not null +# tiktok :string +# timezone :string(500) +# twelve_hour :boolean default(TRUE) +# twitch :string +# twitter :string +# unconfirmed_email :string +# unlock_token :string +# website :string +# willing_to_moderate :boolean default(FALSE) +# year_of_birth :integer +# youtube :string +# created_at :datetime not null +# updated_at :datetime not null +# reg_id :string +# +# Indexes +# +# index_people_on_bio (bio) USING gin +# index_people_on_confirmation_token (confirmation_token) UNIQUE +# index_people_on_name (name) USING gin +# index_people_on_pseudonym (pseudonym) USING gin +# index_people_on_published_name (published_name) USING gin +# index_people_on_reset_password_token (reset_password_token) UNIQUE +# index_people_on_unlock_token (unlock_token) UNIQUE +# +class PersonSyncDatum < Person + has_many :registration_sync_matches, + class_name: 'Registration::RegistrationSyncMatch', + foreign_key: 'pid' + + has_many :registration_sync_data, through: :registration_sync_matches +end diff --git a/app/models/publication_status.rb b/app/models/publication_status.rb index b9a0efe86..57dc5991f 100644 --- a/app/models/publication_status.rb +++ b/app/models/publication_status.rb @@ -1,14 +1,15 @@ # == Schema Information # -# Table name: publication_statuses +# Table name: job_statuses # # id :uuid not null, primary key # lock_version :integer default(0) +# result :jsonb # status :string # submit_time :datetime +# type :string # created_at :datetime not null # updated_at :datetime not null # -class PublicationStatus < ApplicationRecord - validates_inclusion_of :status, in: %w[inprogress completed] +class PublicationStatus < JobStatus end diff --git a/app/models/registration_sync_datum.rb b/app/models/registration_sync_datum.rb index 5289a143c..c377e2700 100644 --- a/app/models/registration_sync_datum.rb +++ b/app/models/registration_sync_datum.rb @@ -4,6 +4,7 @@ # # id :uuid not null, primary key # alternative_email :string +# badge_name :string # email :string # lock_version :integer # name :string @@ -16,8 +17,11 @@ # # Indexes # +# index_registration_sync_data_on_alternative_email (alternative_email) USING gin +# index_registration_sync_data_on_badge_name (badge_name) USING gin # index_registration_sync_data_on_email (email) USING gin # index_registration_sync_data_on_name (name) USING gin +# index_registration_sync_data_on_preferred_name (preferred_name) USING gin # index_registration_sync_data_on_reg_id (reg_id) # index_registration_sync_data_on_registration_number (registration_number) # @@ -26,5 +30,13 @@ class RegistrationSyncDatum < ApplicationRecord class_name: 'Registration::RegistrationSyncMatch', foreign_key: 'rid' + # limit the matches ...? + # Add index of reg_id to people + # where reg_id not in people.reg_id + has_one :matched_person, + class_name: 'Person', + foreign_key: 'reg_id', + primary_key: 'reg_id' + has_many :people, through: :registration_sync_matches end diff --git a/app/models/registration_sync_status.rb b/app/models/registration_sync_status.rb new file mode 100644 index 000000000..f94aa1c97 --- /dev/null +++ b/app/models/registration_sync_status.rb @@ -0,0 +1,15 @@ +# == Schema Information +# +# Table name: job_statuses +# +# id :uuid not null, primary key +# lock_version :integer default(0) +# result :jsonb +# status :string +# submit_time :datetime +# type :string +# created_at :datetime not null +# updated_at :datetime not null +# +class RegistrationSyncStatus < JobStatus +end diff --git a/app/policies/person_sync_datum_policy.rb b/app/policies/person_sync_datum_policy.rb new file mode 100644 index 000000000..445d8fe3e --- /dev/null +++ b/app/policies/person_sync_datum_policy.rb @@ -0,0 +1,16 @@ +class PersonSyncDatumPolicy < PlannerPolicy + + def dismiss_match? + allowed?(action: :dismiss_match) + end + + def match? + allowed?(action: :match) + end + + class Scope < PlannerPolicy::Scope + def resolve + scope.all + end + end +end diff --git a/app/policies/registration_sync_datum_policy.rb b/app/policies/registration_sync_datum_policy.rb new file mode 100644 index 000000000..08513b86b --- /dev/null +++ b/app/policies/registration_sync_datum_policy.rb @@ -0,0 +1,19 @@ +class RegistrationSyncDatumPolicy < PlannerPolicy + def people? + allowed?(action: :people) + end + + def sync_statistics? + allowed?(action: :sync_statistics) + end + + def synchronize? + allowed?(action: :synchronize) + end + + class Scope < PlannerPolicy::Scope + def resolve + scope.all + end + end +end diff --git a/app/serializers/person_serializer.rb b/app/serializers/person_serializer.rb index 3c1bcf29c..d192f2336 100644 --- a/app/serializers/person_serializer.rb +++ b/app/serializers/person_serializer.rb @@ -66,6 +66,8 @@ # published_name :string # published_name_sort_by :string # reddit :string +# reg_attending_status :string +# reg_match :enum default("none") # registered :boolean default(FALSE), not null # registration_number :string # registration_type :string @@ -148,7 +150,10 @@ class PersonSerializer #< ActiveModel::Serializer :excluded_demographic_categories, :global_diaspora, :non_anglophone, - :reg_id + :reg_id, + :reg_attending_status, + :date_reg_synced + :reg_match # status and comments hidden except for staff protected_attributes :con_state, :comments diff --git a/app/serializers/person_sync_datum_serializer.rb b/app/serializers/person_sync_datum_serializer.rb new file mode 100644 index 000000000..b2da13f04 --- /dev/null +++ b/app/serializers/person_sync_datum_serializer.rb @@ -0,0 +1,136 @@ +# == Schema Information +# +# Table name: people +# +# id :uuid not null, primary key +# accommodations :string(10000) +# age_at_convention :string +# attendance_type :string(200) +# availability_notes :string +# bio :text +# black_diaspora :string(10000) +# bsky :string +# can_photo :enum default("no") +# can_photo_exceptions :string(10000) +# can_record :enum default("no") +# can_record_exceptions :string(10000) +# can_share :boolean default(FALSE), not null +# can_stream :enum default("no") +# can_stream_exceptions :string(10000) +# comments :text +# con_state :enum default("not_set") +# confirmation_sent_at :datetime +# confirmation_token :string +# confirmed_at :datetime +# current_sign_in_at :datetime +# current_sign_in_ip :inet +# date_reg_synced :datetime +# demographic_categories :string +# do_not_assign_with :string(10000) +# encrypted_password :string default(""), not null +# ethnicity :string(400) +# excluded_demographic_categories :string +# facebook :string +# failed_attempts :integer default(0), not null +# fediverse :string +# flickr :string +# gender :string(400) +# global_diaspora :string +# indigenous :string(10000) +# instagram :string +# integrations :jsonb not null +# is_local :boolean default(FALSE) +# job_title :string +# language :string(5) default("") +# languages_fluent_in :string(10000) +# last_sign_in_at :datetime +# last_sign_in_ip :inet +# linkedin :string +# lock_version :integer default(0) +# locked_at :datetime +# moderation_experience :string(10000) +# name :string default("") +# name_sort_by :string +# name_sort_by_confirmed :boolean default(FALSE) +# needs_accommodations :boolean default(FALSE) +# non_anglophone :string +# non_us_centric_perspectives :string(10000) +# opted_in :boolean default(FALSE), not null +# organization :string +# othered :string(10000) +# othersocialmedia :string +# pronouns :string(400) +# pseudonym :string +# pseudonym_sort_by :string +# pseudonym_sort_by_confirmed :boolean default(FALSE) +# published_name :string +# published_name_sort_by :string +# reddit :string +# reg_attending_status :string +# reg_match :enum default("none") +# registered :boolean default(FALSE), not null +# registration_number :string +# registration_type :string +# remember_created_at :datetime +# reset_password_sent_at :datetime +# reset_password_token :string +# romantic_sexual_orientation :string +# sign_in_count :integer default(0), not null +# tiktok :string +# timezone :string(500) +# twelve_hour :boolean default(TRUE) +# twitch :string +# twitter :string +# unconfirmed_email :string +# unlock_token :string +# website :string +# willing_to_moderate :boolean default(FALSE) +# year_of_birth :integer +# youtube :string +# created_at :datetime not null +# updated_at :datetime not null +# reg_id :string +# +# Indexes +# +# index_people_on_bio (bio) USING gin +# index_people_on_confirmation_token (confirmation_token) UNIQUE +# index_people_on_name (name) USING gin +# index_people_on_pseudonym (pseudonym) USING gin +# index_people_on_published_name (published_name) USING gin +# index_people_on_reset_password_token (reset_password_token) UNIQUE +# index_people_on_unlock_token (unlock_token) UNIQUE +# +class PersonSyncDatumSerializer + include JSONAPI::Serializer + + attributes :id, :name, :name_sort_by, :name_sort_by_confirmed, + :pseudonym, :pseudonym_sort_by, :pseudonym_sort_by_confirmed, + :published_name, :published_name_sort_by, + :job_title, :organization, :reg_id, + :primary_email, :contact_email, + :reg_match, :date_reg_synced + + has_many :email_addresses, + if: Proc.new { |record, params| AccessControlService.shared_attribute_access?(instance: record, person: params[:current_person]) }, + lazy_load_data: true, serializer: EmailAddressSerializer, + links: { + self: -> (object, params) { + "#{params[:domain]}/person/#{object.id}" + }, + related: -> (object, params) { + "#{params[:domain]}/person/#{object.id}/email_addresses" + } + } + + # The reg data that this person could be matched to + has_many :registration_sync_data, serializer: RegistrationSyncDatumSerializer + # links: { + # self: -> (object, params) { + # "#{params[:domain]}/person_sync_data/#{object.id}" + # }, + # related: -> (object, params) { + # "#{params[:domain]}/registration_sync_datum/#{object.id}/registration_sync_data" + # } + # } +end diff --git a/app/serializers/registration_sync_datum_serializer.rb b/app/serializers/registration_sync_datum_serializer.rb new file mode 100644 index 000000000..b394d3a03 --- /dev/null +++ b/app/serializers/registration_sync_datum_serializer.rb @@ -0,0 +1,38 @@ +# == Schema Information +# +# Table name: registration_sync_data +# +# id :uuid not null, primary key +# alternative_email :string +# badge_name :string +# email :string +# lock_version :integer +# name :string +# preferred_name :string +# raw_info :jsonb not null +# registration_number :string +# created_at :datetime not null +# updated_at :datetime not null +# reg_id :string +# +# Indexes +# +# index_registration_sync_data_on_alternative_email (alternative_email) USING gin +# index_registration_sync_data_on_badge_name (badge_name) USING gin +# index_registration_sync_data_on_email (email) USING gin +# index_registration_sync_data_on_name (name) USING gin +# index_registration_sync_data_on_preferred_name (preferred_name) USING gin +# index_registration_sync_data_on_reg_id (reg_id) +# index_registration_sync_data_on_registration_number (registration_number) +# +class RegistrationSyncDatumSerializer + include JSONAPI::Serializer + + attributes :id, :lock_version, :created_at, :updated_at, + :reg_id, :registration_number, :name, :email, + :preferred_name, :alternative_email, + :badge_name, + :raw_info + + has_one :matched_person, serializer: PersonSerializer +end diff --git a/app/services/identity_service.rb b/app/services/identity_service.rb index 9f9f43d12..659c26cd0 100644 --- a/app/services/identity_service.rb +++ b/app/services/identity_service.rb @@ -105,23 +105,39 @@ def self.clear_person_reg_info(person:) person.registration_type = nil person.registered = false person.registration_number = nil + person.reg_attending_status = nil person.date_reg_synced = Time.now end - def self.update_reg_info(person:, details:) - person.registration_number = details['ticket_number'] - # Based on the products that they have - person.registration_type = details['product'] - person.reg_id = details['id'] - person.registered = true - # Need to store time of last sync - person.date_reg_synced = Time.now - # Attendance type in Plano is one of - # in_person, hybrid, virtual - # Clyde does not map to these well. Recommend that we get this from survey and Person profile - # in Plano instead. - # person.attendance_type = - person.save! + def self.update_reg_info(person:, details:, reg_match: Person.reg_matches[:self]) + # If the Ticket Numbers do not match then we reset cause there may be an issue + if person.registration_number && person.registration_number != details['ticket_number'] + clear_person_reg_info(person: person) + # If the Reg numbers do not match then we reset cause there may be an issue + elsif person.reg_id && person.reg_id != details['id'].to_s + clear_person_reg_info(person: person) + else + person.reg_id = details['id'] unless person.reg_id + person.registration_number = details['ticket_number'] unless person.registration_number + # Based on the products that they have + person.registration_type = details['product'] + person.registered = details['attending_status'] != 'Not Attending' + person.reg_attending_status = details['attending_status'] + # Need to store time of last sync + # How the registration match was done + person.reg_match = reg_match + + # This will ensure update is done only any of the reg data has changed + if person.changed? + person.date_reg_synced = Time.now + end + # Attendance type in Plano is one of + # in_person, hybrid, virtual + # Clyde does not map to these well. Recommend that we get this from survey and Person profile + # in Plano instead. + # person.attendance_type = + end + person.changed? ? person.save! : false end def self.create_identity_from_clyde(details:) @@ -154,15 +170,7 @@ def self.create_identity_from_clyde(details:) identity.save! end - # Attending status does not map well - we will need Plano to ask - # person.attendance_type = details[:attending_status] - - person.registration_number = details['ticket_number'] - # TODO: we need to base this on the products that they have - # which requires a change to the Clyde API to get the products for the person - # person.registration_type = details[:product] - # person.registered = true - person.save! + update_reg_info(person: person, details: details) identity end diff --git a/app/services/publication_service.rb b/app/services/publication_service.rb index fd599a814..6ca5a0f27 100644 --- a/app/services/publication_service.rb +++ b/app/services/publication_service.rb @@ -1,5 +1,6 @@ module PublicationService + # def self.start_publish_job pstatus = PublicationStatus.order('created_at desc').first pstatus = PublicationStatus.new if pstatus == nil diff --git a/app/workers/registration_sync_worker.rb b/app/workers/registration_sync_worker.rb index 5111255c9..f3b98897a 100644 --- a/app/workers/registration_sync_worker.rb +++ b/app/workers/registration_sync_worker.rb @@ -10,70 +10,136 @@ class RegistrationSyncWorker include Sidekiq::Worker def perform - # Phase 1 - get the data from Clyde and store it - phase1 - # Phase 2 - phase2 + PublishedSession.transaction do + # get the data from Clyde and store it + puts "--- Sync Load Phase #{Time.now}" + load_phase(page_size: 500) + puts "--- Update Phase #{Time.now}" + number_updated = update_phase + puts "--- Match Phase #{Time.now}" + number_matched = matched_phase + + # update the status + status = RegistrationSyncStatus.order('created_at desc').first + status = RegistrationSyncStatus.new if status == nil + + status.result = { + updated: number_updated, + matched: number_matched, + not_found: Person.where("reg_id is null").count + } + status.status = 'completed' + status.save! + end + puts "--- Sync Complete #{Time.now}" end # Phase 1 is to suck up the data from Reg and put it into a temp store # for matching - def phase1(page_size: 500) - RegistrationSyncDatum.connection.truncate(RegistrationSyncDatum.table_name) - + def load_phase(page_size: 500) # Get the clyde service and use the AUTH key that we have svc = ClydeService.get_svc(token: ENV['CLYDE_AUTH_KEY']) + if !svc.token + raise "Missing auth token! abort abort abort!" + end + RegistrationSyncDatum.connection.truncate(RegistrationSyncDatum.table_name) results = svc.people_by_page(page: 1, page_size: page_size) + store_reg_data(data: results['data']) - last_page = results['meta']['last_page'].to_i + last_page = results.dig('meta', 'last_page')&.to_i || 0 for page in (2..last_page) do results = svc.people_by_page(page: page, page_size: page_size) - store_reg_data(data: results['data']) + if !results["message"] + store_reg_data(data: results['data']) + else + puts "We had an issue with the Clyde ... people by page #{page}, #{page_size}, #{last_page}, #{results["message"]}" + end end end + def update_phase + number_updated = 0 + Person.transaction do + existing = Person.where("reg_id is not null") + existing.each do |person| + datum = RegistrationSyncDatum.find_by reg_id: person.reg_id + + if datum + res = IdentityService.update_reg_info(person: person, details: datum.raw_info, reg_match: Person.reg_matches[:automatic]) + number_updated += 1 if res + end + end + end + + number_updated + end + # Find good matches and update their information with that from the reg service - def phase2 + def matched_phase # Find all the people that have an unique match for name AND email # (i.e. there is no other match for that registration info) matched = Registration::RegistrationMapCount.where( - "pid in (?)", Registration::RegistrationMapPeopleCount.where("count = 1").pluck(:pid) + "pid in (select pid from registration_map_people_counts where count = 1)" ).where( - "reg_id in (?)", Registration::RegistrationMapRegCount.where("count = 1").pluck(:reg_id) + "reg_id in (select reg_id from registration_map_reg_counts where count = 1)" + ).where( + "pid not in (select id from people where reg_id is not null)" ).where("sub_count = 2") # Update those people with matched information - Person.transaction do - matched.each do |match| + number_matched = 0 + matched.each do |match| + Person.transaction do person = match.person - # If the person has been mapped to another reg then we ignore it - next if (person.reg_id && (person.reg_id != match.reg_id)) - datum = RegistrationSyncDatum.find_by reg_id: match.reg_id + # If the person has already been mapped then we ignore it + if person.reg_id.nil? + datum = RegistrationSyncDatum.find_by reg_id: match.reg_id - IdentityService.update_reg_info(person: person, details: datum.raw_info) - person.save! + # If we match via the worker it is an "automatic match" + IdentityService.update_reg_info(person: person, details: datum.raw_info, reg_match: Person.reg_matches[:automatic]) + number_matched += 1 + end end end + + number_matched end def store_reg_data(data:) RegistrationSyncDatum.transaction do - data.each do |d| - # puts "#{d['id']} -> #{d['full_name']} -> #{d['email']}" - # preferred_name, alternative_email - # TODO: move to an adapter when we have to support multiple reg services - RegistrationSyncDatum.create( - reg_id: d['id'], - name: d['full_name'], - email: d['email'], - registration_number: d['ticket_number'], - preferred_name: d['preferred_name'], - alternative_email: d['alternative_email'], - raw_info: d - ) + if data + data.each do |d| + # puts "#{d['id']} -> #{d['full_name']} -> #{d['email']}" + # preferred_name, alternative_email + # TODO: move to an adapter when we have to support multiple reg services + next unless d['attending_status'] != 'Not Attending' + # Products to exclude from matching + next if [ + 'Chengdu', + 'Volunteer', + 'Apocryphal', + 'Infant', + 'Installment', + 'Hall Pass', + 'Staff', + ].include? d['product_list_name'] + + RegistrationSyncDatum.create( + reg_id: d['id'], + name: d['full_name']&.strip, + email: d['email']&.strip, + registration_number: d['ticket_number']&.strip, + preferred_name: d['preferred_name']&.strip, + alternative_email: d['alternative_email']&.strip, + badge_name: d['badge']&.strip, + raw_info: d + ) + end + else + puts "There was an error! Data is empty" end end end diff --git a/config/routes.rb b/config/routes.rb index a6a1f35ae..5c4550ce6 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -204,6 +204,18 @@ resources :parameter_names, path: 'parameter_name' resources :page_contents, path: 'page_content' + # + get 'registration_sync_data/sync_statistics', to: 'registration_sync_data#sync_statistics' + get 'registration_sync_data/synchronize', to: 'registration_sync_data#synchronize' + resources :registration_sync_data, path: 'registration_sync_datum' do + # This needs to work a bit different than the other sub relationships + get 'people', to: 'registration_sync_data#people' + end + + post 'person_sync_datum/dismiss_match', to: 'person_sync_data#dismiss_match' + post 'person_sync_datum/match', to: 'person_sync_data#match' + resources :person_sync_data, path: 'person_sync_datum' + # Curated tags are the list of tags for a given context etc resources :curated_tags, path: 'curated_tag' diff --git a/db/migrate/20240429160250_add_reg_attending_status_to_person.rb b/db/migrate/20240429160250_add_reg_attending_status_to_person.rb new file mode 100644 index 000000000..a88b37518 --- /dev/null +++ b/db/migrate/20240429160250_add_reg_attending_status_to_person.rb @@ -0,0 +1,5 @@ +class AddRegAttendingStatusToPerson < ActiveRecord::Migration[6.1] + def change + add_column :people, :reg_attending_status, :string, default: nil + end +end diff --git a/db/migrate/20240515001411_create_job_status.rb b/db/migrate/20240515001411_create_job_status.rb new file mode 100644 index 000000000..4c77b163d --- /dev/null +++ b/db/migrate/20240515001411_create_job_status.rb @@ -0,0 +1,15 @@ +class CreateJobStatus < ActiveRecord::Migration[6.1] + def change + rename_table :publication_statuses, :job_statuses + add_column :job_statuses, :type, :string + + reversible do |change| + change.up do + execute <<-SQL + UPDATE job_statuses set type = 'PublicationStatus'; + SQL + end + end + + end +end diff --git a/db/migrate/20240521184252_add_badgename_to_reg_sync_data.rb b/db/migrate/20240521184252_add_badgename_to_reg_sync_data.rb new file mode 100644 index 000000000..f0892bfb8 --- /dev/null +++ b/db/migrate/20240521184252_add_badgename_to_reg_sync_data.rb @@ -0,0 +1,5 @@ +class AddBadgenameToRegSyncData < ActiveRecord::Migration[6.1] + def change + add_column :registration_sync_data, :badge_name, :string, default: nil + end +end diff --git a/db/migrate/20240521193119_add_indexes_to_reg_sync_data.rb b/db/migrate/20240521193119_add_indexes_to_reg_sync_data.rb new file mode 100644 index 000000000..1c9fe1b49 --- /dev/null +++ b/db/migrate/20240521193119_add_indexes_to_reg_sync_data.rb @@ -0,0 +1,7 @@ +class AddIndexesToRegSyncData < ActiveRecord::Migration[6.1] + def change + add_index :registration_sync_data, :badge_name, using: :gin, opclass: :gin_trgm_ops + add_index :registration_sync_data, :preferred_name, using: :gin, opclass: :gin_trgm_ops + add_index :registration_sync_data, :alternative_email, using: :gin, opclass: :gin_trgm_ops + end +end diff --git a/db/migrate/20240522174506_create_dismissed_reg_sync_matches.rb b/db/migrate/20240522174506_create_dismissed_reg_sync_matches.rb new file mode 100644 index 000000000..f72f738df --- /dev/null +++ b/db/migrate/20240522174506_create_dismissed_reg_sync_matches.rb @@ -0,0 +1,13 @@ +class CreateDismissedRegSyncMatches < ActiveRecord::Migration[6.1] + def change + create_table :dismissed_reg_sync_matches, id: :uuid do |t| + t.uuid :person_id, index: true, null: false + t.string :reg_id, index: true, null: false + + t.integer :lock_version + t.timestamps + end + + add_index :dismissed_reg_sync_matches, [:person_id, :reg_id], unique: true, name: "idx_person_reg_id" + end +end diff --git a/db/migrate/20240522190737_add_person_match_enum.rb b/db/migrate/20240522190737_add_person_match_enum.rb new file mode 100644 index 000000000..62900a8b9 --- /dev/null +++ b/db/migrate/20240522190737_add_person_match_enum.rb @@ -0,0 +1,20 @@ +class AddPersonMatchEnum < ActiveRecord::Migration[6.1] + def change + reversible do |change| + change.down do + remove_column :people, :reg_match + + execute <<-SQL + DROP TYPE reg_match_enum; + SQL + end + change.up do + execute <<-SQL + CREATE TYPE reg_match_enum AS ENUM ('none', 'automatic', 'assisted', 'manual', 'self'); + SQL + + add_column :people, :reg_match, :reg_match_enum, default: 'none' + end + end + end +end diff --git a/db/migrate/20240602172220_add_status_info_to_job.rb b/db/migrate/20240602172220_add_status_info_to_job.rb new file mode 100644 index 000000000..3343acc0a --- /dev/null +++ b/db/migrate/20240602172220_add_status_info_to_job.rb @@ -0,0 +1,5 @@ +class AddStatusInfoToJob < ActiveRecord::Migration[6.1] + def change + add_column :job_statuses, :result, :jsonb + end +end diff --git a/db/migrate/20240606115218_ensure_reg_id_unique.rb b/db/migrate/20240606115218_ensure_reg_id_unique.rb new file mode 100644 index 000000000..6f1805d00 --- /dev/null +++ b/db/migrate/20240606115218_ensure_reg_id_unique.rb @@ -0,0 +1,5 @@ +class EnsureRegIdUnique < ActiveRecord::Migration[6.1] + def change + add_index :people, [:reg_id], unique: true, name: "idx_people_reg_id" + end +end diff --git a/db/structure.sql b/db/structure.sql index 5b8418fcc..c4b916a54 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -223,6 +223,19 @@ CREATE TYPE public.phone_type_enum AS ENUM ( ); +-- +-- Name: reg_match_enum; Type: TYPE; Schema: public; Owner: - +-- + +CREATE TYPE public.reg_match_enum AS ENUM ( + 'none', + 'automatic', + 'assisted', + 'manual', + 'self' +); + + -- -- Name: schedule_approval_enum; Type: TYPE; Schema: public; Owner: - -- @@ -666,7 +679,9 @@ END) STORED, global_diaspora character varying, non_anglophone character varying, fediverse character varying, - bsky character varying + bsky character varying, + reg_attending_status character varying, + reg_match public.reg_match_enum DEFAULT 'none'::public.reg_match_enum ); @@ -924,6 +939,20 @@ CREATE TABLE public.curated_tags ( ); +-- +-- Name: dismissed_reg_sync_matches; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.dismissed_reg_sync_matches ( + id uuid DEFAULT public.gen_random_uuid() NOT NULL, + person_id uuid NOT NULL, + reg_id character varying NOT NULL, + lock_version integer, + created_at timestamp(6) without time zone NOT NULL, + updated_at timestamp(6) without time zone NOT NULL +); + + -- -- Name: email_addresses; Type: TABLE; Schema: public; Owner: - -- @@ -1022,6 +1051,22 @@ CREATE TABLE public.integrations ( ); +-- +-- Name: job_statuses; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.job_statuses ( + id uuid DEFAULT public.gen_random_uuid() NOT NULL, + status character varying, + submit_time timestamp without time zone, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL, + lock_version integer DEFAULT 0, + type character varying, + result jsonb +); + + -- -- Name: label_dimensions; Type: TABLE; Schema: public; Owner: - -- @@ -1519,20 +1564,6 @@ CREATE TABLE public.publication_dates ( ); --- --- Name: publication_statuses; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.publication_statuses ( - id uuid DEFAULT public.gen_random_uuid() NOT NULL, - status character varying, - submit_time timestamp without time zone, - created_at timestamp without time zone NOT NULL, - updated_at timestamp without time zone NOT NULL, - lock_version integer DEFAULT 0 -); - - -- -- Name: publish_snapshots; Type: TABLE; Schema: public; Owner: - -- @@ -1610,7 +1641,8 @@ CREATE TABLE public.registration_sync_data ( raw_info jsonb DEFAULT '{}'::jsonb NOT NULL, lock_version integer, created_at timestamp(6) without time zone NOT NULL, - updated_at timestamp(6) without time zone NOT NULL + updated_at timestamp(6) without time zone NOT NULL, + badge_name character varying ); @@ -2438,6 +2470,14 @@ ALTER TABLE ONLY public.curated_tags ADD CONSTRAINT curated_tags_pkey PRIMARY KEY (id); +-- +-- Name: dismissed_reg_sync_matches dismissed_reg_sync_matches_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.dismissed_reg_sync_matches + ADD CONSTRAINT dismissed_reg_sync_matches_pkey PRIMARY KEY (id); + + -- -- Name: email_addresses email_addresses_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -2663,10 +2703,10 @@ ALTER TABLE ONLY public.publication_dates -- --- Name: publication_statuses publication_statuses_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- Name: job_statuses publication_statuses_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- -ALTER TABLE ONLY public.publication_statuses +ALTER TABLE ONLY public.job_statuses ADD CONSTRAINT publication_statuses_pkey PRIMARY KEY (id); @@ -2941,6 +2981,20 @@ CREATE INDEX fk_configurations_parameters_idx ON public.configurations USING btr CREATE UNIQUE INDEX fl_configurations_unique_index ON public.configurations USING btree (parameter); +-- +-- Name: idx_people_reg_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX idx_people_reg_id ON public.people USING btree (reg_id); + + +-- +-- Name: idx_person_reg_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX idx_person_reg_id ON public.dismissed_reg_sync_matches USING btree (person_id, reg_id); + + -- -- Name: idx_tagname_on_context; Type: INDEX; Schema: public; Owner: - -- @@ -3032,6 +3086,20 @@ CREATE INDEX index_audit_survey_versions_on_item_type_and_item_id ON public.audi CREATE INDEX index_convention_roles_on_person_id ON public.convention_roles USING btree (person_id); +-- +-- Name: index_dismissed_reg_sync_matches_on_person_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_dismissed_reg_sync_matches_on_person_id ON public.dismissed_reg_sync_matches USING btree (person_id); + + +-- +-- Name: index_dismissed_reg_sync_matches_on_reg_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_dismissed_reg_sync_matches_on_reg_id ON public.dismissed_reg_sync_matches USING btree (reg_id); + + -- -- Name: index_email_addresses_on_email; Type: INDEX; Schema: public; Owner: - -- @@ -3228,6 +3296,20 @@ CREATE INDEX index_published_programme_item_assignments_on_person_id ON public.p CREATE INDEX index_published_sessions_on_format_id ON public.published_sessions USING btree (format_id); +-- +-- Name: index_registration_sync_data_on_alternative_email; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_registration_sync_data_on_alternative_email ON public.registration_sync_data USING gin (alternative_email public.gin_trgm_ops); + + +-- +-- Name: index_registration_sync_data_on_badge_name; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_registration_sync_data_on_badge_name ON public.registration_sync_data USING gin (badge_name public.gin_trgm_ops); + + -- -- Name: index_registration_sync_data_on_email; Type: INDEX; Schema: public; Owner: - -- @@ -3242,6 +3324,13 @@ CREATE INDEX index_registration_sync_data_on_email ON public.registration_sync_d CREATE INDEX index_registration_sync_data_on_name ON public.registration_sync_data USING gin (name public.gin_trgm_ops); +-- +-- Name: index_registration_sync_data_on_preferred_name; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_registration_sync_data_on_preferred_name ON public.registration_sync_data USING gin (preferred_name public.gin_trgm_ops); + + -- -- Name: index_registration_sync_data_on_reg_id; Type: INDEX; Schema: public; Owner: - -- @@ -3849,6 +3938,14 @@ INSERT INTO "schema_migrations" (version) VALUES ('20240223134703'), ('20240226191153'), ('20240303213410'), -('20240423130325'); +('20240423130325'), +('20240429160250'), +('20240515001411'), +('20240521184252'), +('20240521193119'), +('20240522174506'), +('20240522190737'), +('20240602172220'), +('20240606115218'); diff --git a/lib/tasks/rbac.rake b/lib/tasks/rbac.rake index d27b15314..81126b4fe 100644 --- a/lib/tasks/rbac.rake +++ b/lib/tasks/rbac.rake @@ -302,7 +302,17 @@ namespace :rbac do "index": true, "show": true, "update": false - } + }, + "RegistrationSyncDatum": { + "create": false, + "destroy": false, + "index": false, + "show": false, + "update": false, + "people": false, + "synchronize": false, + "sync_statistics": false + } }) end @@ -643,7 +653,17 @@ namespace :rbac do "index": true, "show": true, "update": false - } + }, + "RegistrationSyncDatum": { + "create": false, + "destroy": false, + "index": true, + "show": true, + "update": false, + "people": true, + "synchronize": false, + "sync_statistics": true + } }) end @@ -984,7 +1004,23 @@ namespace :rbac do "index": true, "show": true, "update": true - } + }, + "RegistrationSyncDatum": { + "create": true, + "destroy": true, + "index": true, + "show": true, + "update": true, + "people": true, + "synchronize": true, + "sync_statistics": true + }, + "PersonSyncDatum": { + "index": true, + "show": true, + "dismiss_match": true, + "match": true + } }) end end diff --git a/spec/models/person_spec.rb b/spec/models/person_spec.rb index 2f62c7000..b83635eb1 100644 --- a/spec/models/person_spec.rb +++ b/spec/models/person_spec.rb @@ -66,6 +66,8 @@ # published_name :string # published_name_sort_by :string # reddit :string +# reg_attending_status :string +# reg_match :enum default("none") # registered :boolean default(FALSE), not null # registration_number :string # registration_type :string diff --git a/test/factories/dismissed_reg_sync_matches.rb b/test/factories/dismissed_reg_sync_matches.rb new file mode 100644 index 000000000..7436a6883 --- /dev/null +++ b/test/factories/dismissed_reg_sync_matches.rb @@ -0,0 +1,22 @@ +# == Schema Information +# +# Table name: dismissed_reg_sync_matches +# +# id :uuid not null, primary key +# lock_version :integer +# created_at :datetime not null +# updated_at :datetime not null +# person_id :uuid not null +# reg_id :string not null +# +# Indexes +# +# idx_person_reg_id (person_id,reg_id) UNIQUE +# index_dismissed_reg_sync_matches_on_person_id (person_id) +# index_dismissed_reg_sync_matches_on_reg_id (reg_id) +# +FactoryBot.define do + factory :dismissed_reg_sync_match do + + end +end diff --git a/test/factories/registration_sync_data.rb b/test/factories/registration_sync_data.rb index 57ac18455..ce5e2faa7 100644 --- a/test/factories/registration_sync_data.rb +++ b/test/factories/registration_sync_data.rb @@ -4,6 +4,7 @@ # # id :uuid not null, primary key # alternative_email :string +# badge_name :string # email :string # lock_version :integer # name :string @@ -16,8 +17,11 @@ # # Indexes # +# index_registration_sync_data_on_alternative_email (alternative_email) USING gin +# index_registration_sync_data_on_badge_name (badge_name) USING gin # index_registration_sync_data_on_email (email) USING gin # index_registration_sync_data_on_name (name) USING gin +# index_registration_sync_data_on_preferred_name (preferred_name) USING gin # index_registration_sync_data_on_reg_id (reg_id) # index_registration_sync_data_on_registration_number (registration_number) # diff --git a/test/models/dismissed_reg_sync_match_test.rb b/test/models/dismissed_reg_sync_match_test.rb new file mode 100644 index 000000000..c923a01d3 --- /dev/null +++ b/test/models/dismissed_reg_sync_match_test.rb @@ -0,0 +1,24 @@ +# == Schema Information +# +# Table name: dismissed_reg_sync_matches +# +# id :uuid not null, primary key +# lock_version :integer +# created_at :datetime not null +# updated_at :datetime not null +# person_id :uuid not null +# reg_id :string not null +# +# Indexes +# +# idx_person_reg_id (person_id,reg_id) UNIQUE +# index_dismissed_reg_sync_matches_on_person_id (person_id) +# index_dismissed_reg_sync_matches_on_reg_id (reg_id) +# +require "test_helper" + +class DismissedRegSyncMatchTest < ActiveSupport::TestCase + # test "the truth" do + # assert true + # end +end diff --git a/test/models/registration_sync_datum_test.rb b/test/models/registration_sync_datum_test.rb index 12f00b5de..b317a6b45 100644 --- a/test/models/registration_sync_datum_test.rb +++ b/test/models/registration_sync_datum_test.rb @@ -4,6 +4,7 @@ # # id :uuid not null, primary key # alternative_email :string +# badge_name :string # email :string # lock_version :integer # name :string @@ -16,8 +17,11 @@ # # Indexes # +# index_registration_sync_data_on_alternative_email (alternative_email) USING gin +# index_registration_sync_data_on_badge_name (badge_name) USING gin # index_registration_sync_data_on_email (email) USING gin # index_registration_sync_data_on_name (name) USING gin +# index_registration_sync_data_on_preferred_name (preferred_name) USING gin # index_registration_sync_data_on_reg_id (reg_id) # index_registration_sync_data_on_registration_number (registration_number) #