From e6d6009bc71fd733adb57b29c34b843bafef70db Mon Sep 17 00:00:00 2001 From: Matthew Whitlock Date: Wed, 19 Jul 2023 07:51:28 -0700 Subject: [PATCH 01/13] Implement error callback pop --- include/fenix.h | 4 +++- include/fenix_process_recovery.h | 2 ++ src/fenix.c | 4 ++++ src/fenix_callbacks.c | 13 +++++++++++++ 4 files changed, 22 insertions(+), 1 deletion(-) diff --git a/include/fenix.h b/include/fenix.h index 1a283bf..20dcdcc 100644 --- a/include/fenix.h +++ b/include/fenix.h @@ -69,7 +69,7 @@ extern "C" { #define FENIX_SUCCESS 0 #define FENIX_ERROR_UNINITIALIZED -9 #define FENIX_ERROR_NOCATEGORY -10 -#define FENIX_ERROR_CALLBACK_NOT_REGISTERD -11 +#define FENIX_ERROR_CALLBACK_NOT_REGISTERED -11 #define FENIX_ERROR_GROUP_CREATE -12 #define FENIX_ERROR_MEMBER_CREATE -13 #define FENIX_ERROR_COMMIT_BARRIER -133 @@ -142,6 +142,8 @@ int Fenix_Initialized(int *); int Fenix_Callback_register(void (*recover)(MPI_Comm, int, void *), void *callback_data); +int Fenix_Callback_pop(); + int Fenix_get_number_of_ranks_with_role(int, int *); int Fenix_get_role(MPI_Comm comm, int rank, int *role); diff --git a/include/fenix_process_recovery.h b/include/fenix_process_recovery.h index bb9d63a..5243ae4 100644 --- a/include/fenix_process_recovery.h +++ b/include/fenix_process_recovery.h @@ -100,6 +100,8 @@ int __fenix_repair_ranks(); int __fenix_callback_register(void (*recover)(MPI_Comm, int, void *), void *); +int __fenix_callback_pop(); + void __fenix_callback_push(fenix_callback_list_t **, fenix_callback_func *); void __fenix_callback_invoke_all(int error); diff --git a/src/fenix.c b/src/fenix.c index 6be875f..a8e5e28 100644 --- a/src/fenix.c +++ b/src/fenix.c @@ -67,6 +67,10 @@ int Fenix_Callback_register(void (*recover)(MPI_Comm, int, void *), void *callba return __fenix_callback_register(recover, callback_data); } +int Fenix_Callback_pop() { + return __fenix_callback_pop(); +} + int Fenix_Initialized(int *flag) { *flag = (fenix.fenix_init_flag) ? 1 : 0; return FENIX_SUCCESS; diff --git a/src/fenix_callbacks.c b/src/fenix_callbacks.c index 885058d..8779402 100644 --- a/src/fenix_callbacks.c +++ b/src/fenix_callbacks.c @@ -80,6 +80,19 @@ int __fenix_callback_register(void (*recover)(MPI_Comm, int, void *), void *call return error_code; } +int __fenix_callback_pop(){ + if(!fenix.fenix_init_flag) return FENIX_ERROR_UNINITIALIZED; + if(fenix.callback_list == NULL) return FENIX_ERROR_CALLBACK_NOT_REGISTERED; + + fenix_callback_list_t* old_head = fenix.callback_list; + fenix.callback_list = old_head->next; + + free(old_head->callback); + free(old_head); + + return FENIX_SUCCESS; +} + void __fenix_callback_invoke_all(int error) { fenix_callback_list_t *current = fenix.callback_list; From fb665da3fd21caea6b0277bba755812d7561e0cc Mon Sep 17 00:00:00 2001 From: Matthew Whitlock Date: Wed, 19 Jul 2023 08:17:35 -0700 Subject: [PATCH 02/13] Bugfixes for failures during data recovery Add a recovery callback to destroy partially-recovered members when interrupted by failure Fix the case of one rank finishing a store where its partner fails, followed by a commit on the succesfull rank. Now come to a consensus on timestamps on group reinitialization. --- src/fenix_data_policy_in_memory_raid.c | 192 ++++++++++++++++++------- src/fenix_data_recovery.c | 6 +- 2 files changed, 144 insertions(+), 54 deletions(-) diff --git a/src/fenix_data_policy_in_memory_raid.c b/src/fenix_data_policy_in_memory_raid.c index ca4b007..6495275 100644 --- a/src/fenix_data_policy_in_memory_raid.c +++ b/src/fenix_data_policy_in_memory_raid.c @@ -56,6 +56,7 @@ #include #include "fenix.h" +#include "fenix_ext.h" #include "fenix_opt.h" #include "fenix_data_subset.h" #include "fenix_data_recovery.h" @@ -123,8 +124,25 @@ typedef struct __fenix_imr_group{ int entries_count; fenix_imr_mentry_t* entries; int num_snapshots; + int* timestamps; } fenix_imr_group_t; +typedef struct __fenix_imr_undo_log{ + int groupid, memberid; +} fenix_imr_undo_log_t; + +void __imr_sync_timestamps(fenix_imr_group_t* group); + +void __imr_undo_restore(MPI_Comm comm, int err, void* data){ + fenix_imr_undo_log_t* undo_log = (fenix_imr_undo_log_t*)data; + + Fenix_Data_member_delete(undo_log->groupid, undo_log->memberid); + + free(data); + Fenix_Callback_pop(); //Should be this callback itself. +} + + void __fenix_policy_in_memory_raid_get_group(fenix_group_t** group, MPI_Comm comm, int timestart, int depth, void* policy_value, int* flag){ *group = (fenix_group_t *)malloc(sizeof(fenix_imr_group_t)); @@ -257,8 +275,11 @@ void __fenix_policy_in_memory_raid_get_group(fenix_group_t** group, MPI_Comm com new_group->entries = (fenix_imr_mentry_t*) malloc(sizeof(fenix_imr_mentry_t) * __FENIX_IMR_DEFAULT_MENTRY_NUM); new_group->num_snapshots = 0; - - + new_group->timestamps = (int*)malloc(sizeof(int)*depth); + + new_group->base.comm = comm; + new_group->base.current_rank = my_rank; + __imr_sync_timestamps(new_group); *flag = FENIX_SUCCESS; } @@ -370,12 +391,7 @@ int __imr_member_create(fenix_group_t* g, fenix_member_entry_t* mentry){ //Initialize to smallest # blocks allowed. __fenix_data_subset_init(1, new_imr_mentry->data_regions + i); new_imr_mentry->data_regions[i].specifier = __FENIX_SUBSET_EMPTY; - - //-1 is not a valid timestamp, use as an indicator that the data isn't valid. - new_imr_mentry->timestamp[i] = -1; } - //The first commit's timestamp is the group's timestart. - new_imr_mentry->timestamp[0] = group->base.timestart; group->entries_count++; @@ -398,7 +414,7 @@ void __imr_member_free(fenix_imr_mentry_t* mentry, int depth){ } int __imr_member_delete(fenix_group_t* g, int member_id){ - int retval = -1; + int retval = FENIX_SUCCESS; fenix_imr_group_t* group = (fenix_imr_group_t*)g; //Find the member first fenix_imr_mentry_t *mentry; @@ -460,9 +476,10 @@ int __imr_member_store(fenix_group_t* g, int member_id, void* recv_buf = malloc(serialized_size * member_data->datatype_size); MPI_Sendrecv(serialized, serialized_size * member_data->datatype_size, MPI_BYTE, - group->partners[1], group->base.groupid ^ STORE_PAYLOAD_TAG, recv_buf, - serialized_size * member_data->datatype_size, MPI_BYTE, group->partners[0], - group->base.groupid ^ STORE_PAYLOAD_TAG, group->base.comm, NULL); + group->partners[1], group->base.groupid ^ STORE_PAYLOAD_TAG, + recv_buf, serialized_size * member_data->datatype_size, MPI_BYTE, + group->partners[0], group->base.groupid ^ STORE_PAYLOAD_TAG, + group->base.comm, NULL); //Expand the serialized data out and store into the partner's portion of this data entry. __fenix_data_subset_deserialize(&subset_specifier, recv_buf, @@ -575,13 +592,18 @@ int __imr_commit(fenix_group_t* g){ fenix_imr_group_t *group = (fenix_imr_group_t*)g; + if(group->num_snapshots == group->base.depth+1){ + //Full of timestamps, remove the oldest and proceed as normal. + memcpy(group->timestamps, group->timestamps+1, group->base.depth); + group->num_snapshots--; + } + group->timestamps[group->num_snapshots++] = group->base.timestamp; + + //For each entry id (eid) for(int eid = 0; eid < group->entries_count; eid++){ fenix_imr_mentry_t *mentry = &group->entries[eid]; - //Two cases for each member entry: - // (1) depth has been reached, shift out the oldest commit - // (2) depth has not been reached, just commit and start filling a new location. if(mentry->current_head == group->base.depth + 1){ //The entry is full, one snapshot should be shifted out. @@ -598,24 +620,11 @@ int __imr_commit(fenix_group_t* g){ mentry->data[group->base.depth + 1] = first_data; mentry->data_regions[group->base.depth + 1].specifier = __FENIX_SUBSET_EMPTY; - mentry->timestamp[group->base.depth + 1] = mentry->timestamp[group->base.depth] + 1; - - } else { - //The entry is not full, just shift the current head. - mentry->current_head++; - - //Everything is initialized to correct values, we just need to provide - //the correct timestamp for the next snapshot. - mentry->timestamp[mentry->current_head] = mentry->timestamp[mentry->current_head-1] + 1; - - if(eid == 0){ - //Only do this once - group->num_snapshots++; - } + mentry->current_head--; } - } - group->base.timestamp = group->entries[0].timestamp[group->entries[0].current_head - 1]; + mentry->timestamp[mentry->current_head++] = group->base.timestamp; + } return to_return; } @@ -698,6 +707,9 @@ int __imr_member_restore(fenix_group_t* g, int member_id, int retval = -1; fenix_imr_group_t* group = (fenix_imr_group_t*)g; + //One-time fix after a reinit. + if(group->base.timestamp == -1 && group->num_snapshots > 0) + group->base.timestamp = group->timestamps[group->num_snapshots-1]; fenix_imr_mentry_t* mentry; //find_mentry returns the error status. We found the member (and corresponding data) if there are no errors. @@ -711,6 +723,8 @@ int __imr_member_restore(fenix_group_t* g, int member_id, int recovery_locally_possible; + fenix_imr_undo_log_t* undo_data; //Used for undoing partial restores interrupted by failures. + if(group->raid_mode == 1){ int my_data_found, partner_data_found; @@ -722,6 +736,7 @@ int __imr_member_restore(fenix_group_t* g, int member_id, MPI_Sendrecv(&found_member, 1, MPI_INT, group->partners[1], PARTNER_STATUS_TAG, &partner_data_found, 1, MPI_INT, group->partners[0], PARTNER_STATUS_TAG, group->base.comm, NULL); + if(found_member && partner_data_found && my_data_found){ //I have my data, and the person who's data I am backing up has theirs. We're good to go. @@ -738,17 +753,6 @@ int __imr_member_restore(fenix_group_t* g, int member_id, if(!partner_data_found) __fenix_data_member_send_metadata(group->base.groupid, member_id, group->partners[0]); - //Now my partner will need all of the entries. First they'll need to know how many snapshots - //to expect. - if(!partner_data_found) - MPI_Send((void*) &(group->num_snapshots), 1, MPI_INT, group->partners[0], - RECOVER_MEMBER_ENTRY_TAG^group->base.groupid, group->base.comm); - - //They also need the timestamps for each snapshot, as well as the value for the next. - if(!partner_data_found) - MPI_Send((void*)mentry->timestamp, group->num_snapshots+1, MPI_INT, group->partners[0], - RECOVER_MEMBER_ENTRY_TAG^group->base.groupid, group->base.comm); - for(int snapshot = 0; snapshot < group->num_snapshots; snapshot++){ //send data region info next if(!partner_data_found) @@ -788,21 +792,22 @@ int __imr_member_restore(fenix_group_t* g, int member_id, __fenix_member_create(group->base.groupid, packet.memberid, NULL, packet.current_count, packet.datatype_size); + //Mark the member for deletion if another failure interrupts recovering fully. + undo_data = (fenix_imr_undo_log_t*)malloc(sizeof(fenix_imr_undo_log_t)); + undo_data->groupid = group->base.groupid; + undo_data->memberid = member_id; + Fenix_Callback_register(__imr_undo_restore, (void*)undo_data); + __imr_find_mentry(group, member_id, &mentry); int member_data_index = __fenix_search_memberid(group->base.member, member_id); member_data = group->base.member->member_entry[member_data_index]; - - MPI_Recv((void*)&(group->num_snapshots), 1, MPI_INT, group->partners[1], - RECOVER_MEMBER_ENTRY_TAG^group->base.groupid, group->base.comm, NULL); - + mentry->current_head = group->num_snapshots; - //We also need to explicitly ask for all timestamps, since user may have deleted some and caused mischief. - MPI_Recv((void*)(mentry->timestamp), group->num_snapshots + 1, MPI_INT, group->partners[1], - RECOVER_MEMBER_ENTRY_TAG^group->base.groupid, group->base.comm, NULL); - //now recover data. for(int snapshot = 0; snapshot < group->num_snapshots; snapshot++){ + mentry->timestamp[snapshot] = group->timestamps[snapshot]; + __fenix_data_subset_free(mentry->data_regions+snapshot); __fenix_data_subset_recv(mentry->data_regions+snapshot, group->partners[1], __IMR_RECOVER_DATA_REGION_TAG ^ group->base.groupid, group->base.comm); @@ -828,11 +833,16 @@ int __imr_member_restore(fenix_group_t* g, int member_id, free(recv_buf); } } + + //Member restored fully, so we don't need to mark it for undoing on failure. + Fenix_Callback_pop(); + free(undo_data); } recovery_locally_possible = found_member || (my_data_found && partner_data_found); - + if(recovery_locally_possible) retval = FENIX_SUCCESS; + } else if (group->raid_mode == 5){ int* set_results = malloc(sizeof(int) * group->set_size); MPI_Allgather((void*)&found_member, 1, MPI_INT, (void*)set_results, 1, MPI_INT, @@ -890,6 +900,13 @@ int __imr_member_restore(fenix_group_t* g, int member_id, __fenix_member_create(group->base.groupid, packet.memberid, NULL, packet.current_count, packet.datatype_size); + //Mark the member for deletion if another failure interrupts recovering fully. + undo_data = (fenix_imr_undo_log_t*)malloc(sizeof(fenix_imr_undo_log_t)); + undo_data->groupid = group->base.groupid; + undo_data->memberid = member_id; + Fenix_Callback_register(__imr_undo_restore, (void*)undo_data); + + __imr_find_mentry(group, member_id, &mentry); int member_data_index = __fenix_search_memberid(group->base.member, member_id); member_data = group->base.member->member_entry[member_data_index]; @@ -956,6 +973,12 @@ int __imr_member_restore(fenix_group_t* g, int member_id, } } + if(!found_member){ + //Member restored fully, so we don't need to mark it for undoing on failure. + Fenix_Callback_pop(); + free(undo_data); + } + } retval = FENIX_SUCCESS; @@ -1032,7 +1055,7 @@ int __imr_member_restore(fenix_group_t* g, int member_id, } //Dont forget to clear the commit buffer - mentry->data_regions[mentry->current_head].specifier = __FENIX_SUBSET_EMPTY; + if(recovery_locally_possible) mentry->data_regions[mentry->current_head].specifier = __FENIX_SUBSET_EMPTY; return retval; @@ -1128,11 +1151,78 @@ int __imr_reinit(fenix_group_t* g, int* flag){ MPI_Comm_create_group(g->comm, set_group, 0, &(group->set_comm)); } + __imr_sync_timestamps(group); + *flag = FENIX_SUCCESS; return FENIX_SUCCESS; } +void __imr_sync_timestamps(fenix_imr_group_t* group){ + int n_snapshots = group->num_snapshots; + + if(group->raid_mode == 1){ + int partner_snapshots; + MPI_Sendrecv(&n_snapshots, 1, MPI_INT, group->partners[0], 34560, + &partner_snapshots, 1, MPI_INT, group->partners[1], 34560, + group->base.comm, MPI_STATUS_IGNORE); + n_snapshots = n_snapshots > partner_snapshots ? n_snapshots : partner_snapshots; + + MPI_Sendrecv(&n_snapshots, 1, MPI_INT, group->partners[1], 34561, + &partner_snapshots, 1, MPI_INT, group->partners[0], 34561, + group->base.comm, MPI_STATUS_IGNORE); + n_snapshots = n_snapshots > partner_snapshots ? n_snapshots : partner_snapshots; + } else { + MPI_Allreduce(MPI_IN_PLACE, &n_snapshots, 1, MPI_INT, MPI_MAX, group->set_comm); + } + + bool need_reset = group->num_snapshots != n_snapshots; + for(int i = group->num_snapshots; i < n_snapshots; i++) group->timestamps[i] = -1; + + if(group->raid_mode == 1){ + int* p0_stamps = (int*)malloc(sizeof(int)*n_snapshots); + int* p1_stamps = (int*)malloc(sizeof(int)*n_snapshots); + + MPI_Sendrecv(group->timestamps, n_snapshots, MPI_INT, group->partners[1], 34562, + p0_stamps, n_snapshots, MPI_INT, group->partners[0], 34562, + group->base.comm, MPI_STATUS_IGNORE); + MPI_Sendrecv(group->timestamps, n_snapshots, MPI_INT, group->partners[0], 34563, + p1_stamps, n_snapshots, MPI_INT, group->partners[1], 34563, + group->base.comm, MPI_STATUS_IGNORE); + + for(int i = 0; i < n_snapshots; i++){ + int old_stamp = group->timestamps[i]; + group->timestamps[i] = group->timestamps[i] > p0_stamps[i] ? group->timestamps[i] : p0_stamps[i]; + group->timestamps[i] = group->timestamps[i] > p1_stamps[i] ? group->timestamps[i] : p1_stamps[i]; + + need_reset |= group->timestamps[i] != old_stamp; + } + + free(p0_stamps); + free(p1_stamps); + } else { + MPI_Allreduce(MPI_IN_PLACE, group->timestamps, n_snapshots, MPI_INT, MPI_MAX, group->set_comm); + } + + group->num_snapshots = n_snapshots; + if(n_snapshots > 0) group->base.timestamp = group->timestamps[n_snapshots-1]; + else group->base.timestamp = -1; + + //Now fix members + if(need_reset && group->entries_count > 0) { + if(fenix.options.verbose == 1){ + verbose_print("Outdated timestamps on rank %d. All members will require full recovery.\n", + group->base.current_rank); + } + //For now, just delete all members and assume partner(s) can + //help me rebuild fully consistent state + for(int i = group->entries_count-1; i >= 0; i--){ + int memberid = group->entries[i].memberid; + Fenix_Data_member_delete(group->base.groupid, memberid); + } + } +} + int __imr_get_redundant_policy(fenix_group_t* group, int* policy_name, void* policy_value, int* flag){ int retval = FENIX_SUCCESS; diff --git a/src/fenix_data_recovery.c b/src/fenix_data_recovery.c index 7542cce..90b8d5b 100644 --- a/src/fenix_data_recovery.c +++ b/src/fenix_data_recovery.c @@ -551,11 +551,11 @@ int __fenix_data_commit(int groupid, int *timestamp) { } else { fenix_group_t *group = (fenix.data_recovery->group[group_index]); - group->vtbl.commit(group); - - if (group->timestamp +1 -1) group->timestamp++; + if (group->timestamp != -1) group->timestamp++; else group->timestamp = group->timestart; + group->vtbl.commit(group); + if (timestamp != NULL) { *timestamp = group->timestamp; } From e39a028e963f61940fa99e2f12743b430a331735 Mon Sep 17 00:00:00 2001 From: Matthew Whitlock Date: Wed, 19 Jul 2023 10:12:27 -0700 Subject: [PATCH 03/13] Fix deadlock when a rank fails after only some ranks reach Fenix_Finalize More thought can be put in to this (e.g. if a rank has failed, but all remaining ranks reach finalize, could we just finalize anyway?) --- src/fenix_process_recovery.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/fenix_process_recovery.c b/src/fenix_process_recovery.c index b845fa6..0979539 100644 --- a/src/fenix_process_recovery.c +++ b/src/fenix_process_recovery.c @@ -707,18 +707,14 @@ void __fenix_postinit(int *error) void __fenix_finalize() { + MPI_Barrier(*fenix.user_world); + // Any MPI communication call needs to be protected in case they // fail. In that case, we need to recursively call fenix_finalize. // By setting fenix.finalized to 1 we are skipping the longjump // after recovery. fenix.finalized = 1; - int ret = MPI_Barrier( fenix.new_world ); - if (ret != MPI_SUCCESS) { - __fenix_finalize(); - return; - } - if (__fenix_get_current_rank(*fenix.world) == 0) { int spare_rank; MPI_Comm_size(*fenix.world, &spare_rank); @@ -735,7 +731,7 @@ void __fenix_finalize() } } - ret = MPI_Barrier(*fenix.world); + int ret = MPI_Barrier(*fenix.world); if (ret != MPI_SUCCESS) { __fenix_finalize(); return; From 1b04ad0e8379e7a8ab99eead265457e23b4507fb Mon Sep 17 00:00:00 2001 From: Matthew Whitlock Date: Thu, 24 Aug 2023 07:48:06 -0700 Subject: [PATCH 04/13] Fix timestamp bug in commit_barrier --- src/fenix_data_recovery.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/fenix_data_recovery.c b/src/fenix_data_recovery.c index 90b8d5b..314df0b 100644 --- a/src/fenix_data_recovery.c +++ b/src/fenix_data_recovery.c @@ -601,6 +601,8 @@ int __fenix_data_commit_barrier(int groupid, int *timestamp) { fenix.ignore_errs = old_failure_handling; if(can_commit == 1){ + if (group->timestamp != -1) group->timestamp++; + else group->timestamp = group->timestart; retval = group->vtbl.commit(group); } From 1fd2785ed93394638a120a6ca76a2a2a2203f708 Mon Sep 17 00:00:00 2001 From: Matthew Whitlock Date: Thu, 24 Aug 2023 07:48:41 -0700 Subject: [PATCH 05/13] Fix rare deadlock in finalize Possible when a rank fails after reaching finalize if the remaining ranks inconsistently succeed/fail on the barrier finishing. --- include/fenix.h | 3 +- src/fenix_process_recovery.c | 93 +++++++++++++++++++++++++----------- 2 files changed, 68 insertions(+), 28 deletions(-) diff --git a/include/fenix.h b/include/fenix.h index 20dcdcc..3ecb938 100644 --- a/include/fenix.h +++ b/include/fenix.h @@ -105,7 +105,8 @@ extern "C" { #define FENIX_DATA_SUBSET_CREATED 2 #define FENIX_ERRHANDLER_LOC 1 -#define FENIX_DATA_COMMIT_BARRIER_LOC 2 +#define FENIX_FINALIZE_LOC 2 +#define FENIX_DATA_COMMIT_BARRIER_LOC 4 #define FENIX_DATA_POLICY_IN_MEMORY_RAID 13 diff --git a/src/fenix_process_recovery.c b/src/fenix_process_recovery.c index 0979539..52b57ff 100644 --- a/src/fenix_process_recovery.c +++ b/src/fenix_process_recovery.c @@ -220,12 +220,14 @@ int __fenix_preinit(int *role, MPI_Comm comm, MPI_Comm *new_comm, int *argc, cha __fenix_get_current_rank(*fenix.world), fenix.role); } __fenix_finalize_spare(); - } else { + } else if(ret == MPI_ERR_REVOKED){ fenix.repair_result = __fenix_repair_ranks(); if (fenix.options.verbose == 0) { verbose_print("spare rank exiting from MPI_Recv - repair ranks; rank: %d, role: %d\n", __fenix_get_current_rank(*fenix.world), fenix.role); } + } else { + MPIX_Comm_ack_failed(*fenix.world, __fenix_get_world_size(*fenix.world), &a); } fenix.role = FENIX_ROLE_RECOVERED_RANK; } @@ -707,35 +709,53 @@ void __fenix_postinit(int *error) void __fenix_finalize() { - MPI_Barrier(*fenix.user_world); + int location = FENIX_FINALIZE_LOC; + MPIX_Comm_agree(*fenix.user_world, &location); + if(location != FENIX_FINALIZE_LOC){ + //Some ranks are in error recovery, so trigger error handling. + MPIX_Comm_revoke(*fenix.user_world); + MPI_Barrier(*fenix.user_world); + + //In case no-jump enabled after recovery + return __fenix_finalize(); + } - // Any MPI communication call needs to be protected in case they - // fail. In that case, we need to recursively call fenix_finalize. - // By setting fenix.finalized to 1 we are skipping the longjump - // after recovery. - fenix.finalized = 1; - - if (__fenix_get_current_rank(*fenix.world) == 0) { - int spare_rank; - MPI_Comm_size(*fenix.world, &spare_rank); - spare_rank--; - int a; - int i; - for (i = 0; i < fenix.spare_ranks; i++) { - int ret = MPI_Send(&a, 1, MPI_INT, spare_rank, 1, *fenix.world); - if (ret != MPI_SUCCESS) { - __fenix_finalize(); - return; + int first_spare_rank = __fenix_get_world_size(*fenix.user_world); + int last_spare_rank = __fenix_get_world_size(*fenix.world) - 1; + + //If we've reached here, we will finalized regardless of further errors. + fenix.ignore_errs = 1; + while(!fenix.finalized){ + int user_rank = __fenix_get_current_rank(*fenix.user_world); + + if (user_rank == 0) { + for (int i = first_spare_rank; i <= last_spare_rank; i++) { + //We don't care if a spare failed, ignore return value + int unused; + MPI_Send(&unused, 1, MPI_INT, i, 1, *fenix.world); } - spare_rank--; } - } - int ret = MPI_Barrier(*fenix.world); - if (ret != MPI_SUCCESS) { - __fenix_finalize(); - return; + //We need to confirm that rank 0 didn't fail, since it could have + //failed before notifying some spares to leave. + int need_retry = user_rank == 0 ? 0 : 1; + MPIX_Comm_agree(*fenix.user_world, &need_retry); + if(need_retry == 1){ + //Rank 0 didn't contribute, so we need to retry. + MPIX_Comm_shrink(*fenix.user_world, fenix.user_world); + continue; + } else { + //If rank 0 did contribute, we know sends made it, and regardless + //of any other failures we finalize. + fenix.finalized = 1; + } } + + //Now we do one last agree w/ the spares to let them know they can actually + //finalize + int unused; + MPIX_Comm_agree(*fenix.world, &unused); + MPI_Op_free( &fenix.agree_op ); MPI_Comm_set_errhandler( *fenix.world, MPI_ERRORS_ARE_FATAL ); @@ -759,8 +779,27 @@ void __fenix_finalize() void __fenix_finalize_spare() { fenix.fenix_init_flag = 0; - int ret = PMPI_Barrier(*fenix.world); - if (ret != MPI_SUCCESS) { debug_print("MPI_Barrier: %d\n", ret); } + + int unused; + MPI_Request agree_req, recv_req = MPI_REQUEST_NULL; + + MPIX_Comm_iagree(*fenix.world, &unused, &agree_req); + while(true){ + int completed = 0; + MPI_Test(&agree_req, &completed, MPI_STATUS_IGNORE); + if(completed) break; + + int ret = MPI_Test(&recv_req, &completed, MPI_STATUS_IGNORE); + if(completed){ + //We may get duplicate messages informing us to exit + MPI_Irecv(&unused, 1, MPI_INT, MPI_ANY_SOURCE, MPI_ANY_TAG, *fenix.world, &recv_req); + } + if(ret != MPI_SUCCESS){ + MPIX_Comm_ack_failed(*fenix.world, __fenix_get_world_size(*fenix.world), &unused); + } + } + + MPI_Cancel(&recv_req); MPI_Op_free(&fenix.agree_op); MPI_Comm_set_errhandler(*fenix.world, MPI_ERRORS_ARE_FATAL); From 57971d26429b6b524553d79e9e4081ce74bcee6e Mon Sep 17 00:00:00 2001 From: Matthew Whitlock Date: Thu, 24 Aug 2023 12:40:49 -0700 Subject: [PATCH 06/13] Add Fenix_Process_detect_failures --- include/fenix.h | 2 ++ include/fenix_ext.h | 3 +++ include/fenix_process_recovery.h | 2 ++ src/fenix.c | 4 ++++ src/fenix_process_recovery.c | 20 ++++++++++++++++++++ 5 files changed, 31 insertions(+) diff --git a/include/fenix.h b/include/fenix.h index 3ecb938..77d573b 100644 --- a/include/fenix.h +++ b/include/fenix.h @@ -231,6 +231,8 @@ int Fenix_Process_fail_list(int** fail_list); int Fenix_check_cancelled(MPI_Request *request, MPI_Status *status); +int Fenix_Process_detect_failures(int do_recovery); + #if defined(c_plusplus) || defined(__cplusplus) } #endif diff --git a/include/fenix_ext.h b/include/fenix_ext.h index fd4b1a6..ef4dcc4 100644 --- a/include/fenix_ext.h +++ b/include/fenix_ext.h @@ -96,6 +96,9 @@ typedef struct { //Manage state of the comms. Necessary when failures happen rapidly, mussing up state int new_world_exists, user_world_exists; + int dummy_recv_buffer; + MPI_Request check_failures_req; + MPI_Op agree_op; // This is reserved for the global agreement call for Fenix data recovery API diff --git a/include/fenix_process_recovery.h b/include/fenix_process_recovery.h index 5243ae4..9b85e04 100644 --- a/include/fenix_process_recovery.h +++ b/include/fenix_process_recovery.h @@ -118,6 +118,8 @@ void __fenix_set_rank_role(int FenixRankRole); void __fenix_postinit(int *); +int __fenix_detect_failures(int do_recovery); + void __fenix_finalize(); void __fenix_finalize_spare(); diff --git a/src/fenix.c b/src/fenix.c index a8e5e28..70c55d5 100644 --- a/src/fenix.c +++ b/src/fenix.c @@ -209,3 +209,7 @@ int Fenix_check_cancelled(MPI_Request *request, MPI_Status *status){ //Request was (potentially) cancelled if ret is MPI_ERR_PROC_FAILED return ret == MPI_ERR_PROC_FAILED || ret == MPI_ERR_REVOKED; } + +int Fenix_Process_detect_failures(int do_recovery){ + return __fenix_detect_failures(do_recovery); +} diff --git a/src/fenix_process_recovery.c b/src/fenix_process_recovery.c index 52b57ff..8d746d4 100644 --- a/src/fenix_process_recovery.c +++ b/src/fenix_process_recovery.c @@ -686,6 +686,11 @@ void __fenix_postinit(int *error) // fenix.role); //} + if(fenix.new_world_exists){ + //Set up dummy irecv to use for checking for failures. + MPI_Irecv(&fenix.dummy_recv_buffer, 1, MPI_INT, MPI_ANY_SOURCE, + 34095347, fenix.new_world, &fenix.check_failures_req); + } if (fenix.repair_result != 0) { *error = fenix.repair_result; @@ -707,6 +712,21 @@ void __fenix_postinit(int *error) } } +int __fenix_detect_failures(int do_recovery){ + if(!fenix.new_world_exists) return FENIX_ERROR_UNINITIALIZED; + + int old_ignore_errs = fenix.ignore_errs; + fenix.ignore_errs = !do_recovery; + + int req_completed; + int ret = MPI_Test(&fenix.check_failures_req, &req_completed, MPI_STATUS_IGNORE); + + if(req_completed) ret = FENIX_ERROR_INTERN; + + fenix.ignore_errs = old_ignore_errs; + return ret; +} + void __fenix_finalize() { int location = FENIX_FINALIZE_LOC; From 46e627f0bb5566ad9299253ff37e44f82e0ac4c2 Mon Sep 17 00:00:00 2001 From: Matthew Whitlock Date: Thu, 26 Sep 2024 10:58:48 -0700 Subject: [PATCH 07/13] Fix possible request_cancel on MPI_REQUEST_NULL --- src/fenix_process_recovery.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fenix_process_recovery.c b/src/fenix_process_recovery.c index 8d746d4..02f7801 100644 --- a/src/fenix_process_recovery.c +++ b/src/fenix_process_recovery.c @@ -819,7 +819,7 @@ void __fenix_finalize_spare() } } - MPI_Cancel(&recv_req); + if(recv_req != MPI_REQUEST_NULL) MPI_Cancel(&recv_req); MPI_Op_free(&fenix.agree_op); MPI_Comm_set_errhandler(*fenix.world, MPI_ERRORS_ARE_FATAL); From cb4dc6b1f3057d333e52ff2b7f033abce69b19e9 Mon Sep 17 00:00:00 2001 From: Matthew Whitlock Date: Thu, 26 Sep 2024 11:10:18 -0700 Subject: [PATCH 08/13] Fix void* arithmetic --- src/fenix_data_policy_in_memory_raid.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fenix_data_policy_in_memory_raid.c b/src/fenix_data_policy_in_memory_raid.c index 6495275..8eaa362 100644 --- a/src/fenix_data_policy_in_memory_raid.c +++ b/src/fenix_data_policy_in_memory_raid.c @@ -483,7 +483,7 @@ int __imr_member_store(fenix_group_t* g, int member_id, //Expand the serialized data out and store into the partner's portion of this data entry. __fenix_data_subset_deserialize(&subset_specifier, recv_buf, - mentry->data[mentry->current_head] + member_data->datatype_size*member_data->current_count, + ((uint8_t*)mentry->data[mentry->current_head]) + member_data->datatype_size*member_data->current_count, member_data->current_count, member_data->datatype_size); free(recv_buf); @@ -537,7 +537,7 @@ int __imr_member_store(fenix_group_t* g, int member_id, offset = 0; } - MPI_Reduce((void*)((char*)data_buf) + offset, parity_buf, parity_size + (i < remainder ? 1 : 0), MPI_BYTE, + MPI_Reduce((char*)data_buf + offset, parity_buf, parity_size + (i < remainder ? 1 : 0), MPI_BYTE, MPI_BXOR, i, group->set_comm); if(i != my_set_rank){ offset += parity_size + (i < remainder ? 1 : 0); From 5975281b232284a9b463a57f819b92d0afa5cfda Mon Sep 17 00:00:00 2001 From: Matthew Whitlock Date: Thu, 26 Sep 2024 11:17:49 -0700 Subject: [PATCH 09/13] Resolve int-to-ptr warnings --- test/failed_spares/fenix_failed_spares.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/failed_spares/fenix_failed_spares.c b/test/failed_spares/fenix_failed_spares.c index 02d18a4..bea1dd7 100644 --- a/test/failed_spares/fenix_failed_spares.c +++ b/test/failed_spares/fenix_failed_spares.c @@ -66,7 +66,7 @@ const int kKillID = 1; void* exitThread(void* should_exit){ sleep(1); - if( ((int)should_exit) == 1){ + if( ((intptr_t)should_exit) == 1){ pid_t pid = getpid(); kill(pid, SIGTERM); } @@ -92,7 +92,7 @@ int main(int argc, char **argv) { MPI_Comm_size(world_comm, &old_world_size); MPI_Comm_rank(world_comm, &old_rank); - int should_cancel = 0; + intptr_t should_cancel = 0; for(int i = 2; i < argc; i++){ if(atoi(argv[i]) == old_rank) should_cancel = 1; } From 848e50e6f15291958f4e4a77352c03cd2634f8c6 Mon Sep 17 00:00:00 2001 From: Matthew Whitlock Date: Wed, 13 Nov 2024 18:07:06 -0500 Subject: [PATCH 10/13] Swap to Spack and matrix-based GHA for more reliability --- .github/Dockerfile | 29 -------- .github/docker-compose.yml | 118 +++++++++++-------------------- .github/spack.yaml | 31 ++++++++ .github/workflows/ci_checks.yaml | 76 +++++++++++++++----- 4 files changed, 134 insertions(+), 120 deletions(-) delete mode 100644 .github/Dockerfile create mode 100644 .github/spack.yaml diff --git a/.github/Dockerfile b/.github/Dockerfile deleted file mode 100644 index 75e9f40..0000000 --- a/.github/Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -#Built for testing, not designed for application use. - -FROM ubuntu:20.04 -#="open-mpi/ompi" for github.com/open-mpi/ompi -ARG OPENMPI_REPO="open-mpi/ompi" -#="tags" or ="heads", for tag or branch name -ARG OPENMPI_VERS_PREFIX="tags" -#="v5.0.0rc10" or ="v5.0.x", ie tag name or branch name. -ARG OPENMPI_VERS="v5.0.0rc10" -run echo Using https://github.com/${OPENMPI_REPO}/git/refs/${OPENMPI_VERS_PREFIX}/${OPENMPI_VERS} - -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential python3 m4 autoconf automake libtool flex git zlib1g-dev - -#Add files listing latest commit for this branch/tag, which invalidates the clone -#when a change has been pushed. -ADD https://api.github.com/repos/${OPENMPI_REPO}/git/refs/${OPENMPI_VERS_PREFIX}/${OPENMPI_VERS} commit_info -RUN git clone --recursive --branch ${OPENMPI_VERS} --depth 1 https://github.com/${OPENMPI_REPO}.git ompi_src && \ - mkdir ompi_build ompi_install && cd ompi_src && export AUTOMAKE_JOBS=8 && ./autogen.pl && cd ../ompi_build && ../ompi_src/configure --prefix=/ompi_install --disable-man-pages --with-ft=ulfm && make install -j8 && cd .. - - -#New build stage, tosses out src/build trees from openmpi -FROM ubuntu:20.04 -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential cmake ssh zlib1g-dev -COPY . ./fenix_src -COPY --from=0 ompi_install/ /ompi_install/ -ENV PATH="$PATH:/ompi_install/bin" -RUN mkdir fenix_build fenix_install && cd fenix_build && cmake ../fenix_src -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=/ompi_install/bin/mpicc \ - -DFENIX_EXAMPLES=ON -DFENIX_TESTS=ON -DCMAKE_INSTALL_PREFIX=../fenix_install -DMPIEXEC_PREFLAGS="--allow-run-as-root;--map-by;:OVERSUBSCRIBE" && make install -j8 -CMD ["sh", "-c", "cd fenix_build && ctest --verbose --timeout 60"] diff --git a/.github/docker-compose.yml b/.github/docker-compose.yml index b29e083..c8c9024 100644 --- a/.github/docker-compose.yml +++ b/.github/docker-compose.yml @@ -1,81 +1,49 @@ -version: "3.9" - -x-fenix: &fenix - build: &fenix-build - context: ./ - dockerfile: .github/Dockerfile - args: - OPENMPI_REPO: open-mpi/ompi - OPENMPI_VERS_PREFIX: tags - OPENMPI_VERS: v5.0.0rc10 - #Caches should be manually scoped, or they'll conflict. - x-bake: - cache-from: - - type=gha,scope=default - cache-to: - - type=gha,scope=default,mode=max - services: - #fenix_ompi_5rc10: - # <<: *fenix - # image: "fenix:ompi_5rc10" - # build: - # <<: *fenix-build - # x-bake: - # cache-from: - # - type=gha,scope=ompi_5rc10 - # cache-to: - # - type=gha,scope=ompi_5rc10,mode=max - - fenix_ompi_5: - <<: *fenix - image: "fenix:ompi_5" + bootstrap: + image: "bootstrap" build: - <<: *fenix-build + dockerfile_inline: | + FROM spack/ubuntu-jammy:0.22.2 + VOLUME /configs + ARG OMPI_VERSION + ENV OMPI_VERSION=$${OMPI_VERSION} + CMD cp /configs/spack.yaml . && \ + spack -e . add openmpi@$${OMPI_VERSION} && \ + spack -e . containerize >/configs/spack.Dockerfile args: - - OPENMPI_VERS_PREFIX=heads - - OPENMPI_VERS=v5.0.x - x-bake: - cache-from: - - type=gha,scope=ompi_5 - cache-to: - - type=gha,scope=ompi_5,mode=max - - fenix_ompi_main: - <<: *fenix - image: "fenix:ompi_main" + OMPI_VERSION: main + no_cache: true + pull_policy: build + volumes: + - .github/:/configs + + env: + image: "ghcr.io/sandialabs/fenix/env:main" build: - <<: *fenix-build - args: - - OPENMPI_VERS_PREFIX=heads - - OPENMPI_VERS=main - x-bake: - cache-from: - - type=gha,scope=ompi_main - cache-to: - - type=gha,scope=ompi_main,mode=max - - fenix_icldisco_latest: - <<: *fenix - image: "fenix:icldisco_latest" + # Generated by running the bootstrap image + dockerfile: .github/spack.Dockerfile + + fenix: + image: "fenix" build: - <<: *fenix-build + dockerfile_inline: | + ARG OMPI_VERSION main + FROM ghcr.io/sandialabs/fenix/env:$${OMPI_VERSION} + COPY . /fenix + RUN . /opt/spack-environment/activate.sh && \ + mkdir -p /fenix/build && \ + cd /fenix/build && \ + cmake /fenix \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER=mpicc \ + -DFENIX_EXAMPLES=ON \ + -DFENIX_TESTS=ON \ + -DMPIEXEC_PREFLAGS="--allow-run-as-root;--map-by;:oversubscribe" && \ + make -j + + WORKDIR /fenix/build + ENTRYPOINT ["/entrypoint.sh"] + CMD ["ctest", "--output-on-failure", "--timeout", "60"] args: - - OPENMPI_REPO=icldisco/ompi - - OPENMPI_VERS_PREFIX=heads - - OPENMPI_VERS=ulfm/latest - x-bake: - cache-from: - - type=gha,scope=icldisco_latest - cache-to: - - type=gha,scope=icldisco_latest,mode=max - - #fenix_icldisco_experimental: - # <<: *fenix - # image: fenix/icldisco - # build: - # <<: *fenix-build - # args: - # - OPENMPI_REPO=icldisco/ompi - # - OPENMPI_VERS_PREFIX=heads - # - OPENMPI_VERS=ulfm/experimental + OMPI_VERSION: main + pull_policy: build diff --git a/.github/spack.yaml b/.github/spack.yaml new file mode 100644 index 0000000..c5d3611 --- /dev/null +++ b/.github/spack.yaml @@ -0,0 +1,31 @@ +spack: + packages: + openmpi: + variants: +internal-hwloc +internal-libevent +internal-pmix + concretizer: + unify: true + reuse: true + + container: + format: docker + strip: false + images: + os: ubuntu:22.04 + spack: 0.22.2 + os_packages: + build: + - build-essential + - autotools-dev + - pkg-config + - python3 + - m4 + - autoconf + - automake + - flex + - git + - zlib1g-dev + - libperl-dev + - numactl + final: + - build-essential + - cmake diff --git a/.github/workflows/ci_checks.yaml b/.github/workflows/ci_checks.yaml index ebeeef8..14c376c 100644 --- a/.github/workflows/ci_checks.yaml +++ b/.github/workflows/ci_checks.yaml @@ -11,21 +11,65 @@ on: jobs: test: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + ompi_version: + - main + - 5.0.3 + steps: - - uses: actions/checkout@v3 - - uses: docker/setup-buildx-action@v2 - - name: Build - uses: docker/bake-action@master + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Log in to GHCR container registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Bake the bootstrap docker image + uses: docker/bake-action@v5 with: - files: | - .github/docker-compose.yml - load: true - - name: Test open-mpi v5.0.x - if: success() || failure() - run: docker run fenix:ompi_5 - - name: Test open-mpi main - if: success() || failure() - run: docker run fenix:ompi_main - - name: Test icldisco latest - if: success() || failure() - run: docker run fenix:icldisco_latest + files: .github/docker-compose.yml + targets: bootstrap + workdir: . + set: | + *.output=type=docker,name=bootstrap + *.cache-from=type=gha,scope=bootstrap/${{ matrix.ompi_version }} + *.cache-to=type=gha,mode=max,scope=bootstrap/${{ matrix.ompi_version }} + *.args.OMPI_VERSION=${{ matrix.ompi_version }} + + - name: Bootstrap the environment Dockerfile + run: docker run -v ${GITHUB_WORKSPACE}/.github:/configs bootstrap + + - name: Build the environment + uses: docker/bake-action@v5 + with: + files: .github/docker-compose.yml + targets: env + workdir: . + pull: true + set: | + *.cache-from=type=gha,scope=env/${{ matrix.ompi_version }} + *.cache-to=type=gha,mode=max,scope=env/${{ matrix.ompi_version }} + env.tags=ghcr.io/sandialabs/fenix/env:${{ matrix.ompi_version }} + env.output=type=registry,name=ghcr.io/sandialabs/fenix/env:${{ matrix.ompi_version }} + + - name: Build Fenix + uses: docker/bake-action@v5 + with: + source: . + files: .github/docker-compose.yml + targets: fenix + workdir: . + set: | + *.output=type=docker,name=fenix + *.args.OMPI_VERSION=${{ matrix.ompi_version }} + + - name: Test Fenix + run: docker run fenix From f5447ef621a3bc0b050f939654265c58cb5932db Mon Sep 17 00:00:00 2001 From: Matthew Whitlock Date: Thu, 14 Nov 2024 12:40:55 -0500 Subject: [PATCH 11/13] Only rebuild images on request, or once 14 days old --- .github/workflows/build-env/action.yml | 91 ++++++++++++++++++++++++++ .github/workflows/ci_checks.yaml | 53 +++------------ 2 files changed, 100 insertions(+), 44 deletions(-) create mode 100644 .github/workflows/build-env/action.yml diff --git a/.github/workflows/build-env/action.yml b/.github/workflows/build-env/action.yml new file mode 100644 index 0000000..28e16d6 --- /dev/null +++ b/.github/workflows/build-env/action.yml @@ -0,0 +1,91 @@ +name: Build Environment Image +description: Build the Open MPI environment image for Fenix + +inputs: + ompi_version: + description: "Open MPI version to build" + type: string + required: true + token: + description: "GitHub token for logging into GHCR" + type: string + required: true + max_age: + description: "Maximum image age before rebuild, in days" + type: number + required: false + default: 14 + +runs: + using: "composite" + steps: + - name: Check for valid image + shell: bash + run: | + set +e + IMG=ghcr.io/sandialabs/fenix/env:${{ inputs.ompi_version }} + echo "IMG=$IMG" >> $GITHUB_ENV + + docker image rm -f $IMG 2>/dev/null + docker pull $IMG >/dev/null 2>&1 + IMG_CREATED=$(docker inspect --type=image --format '{{.Created}}' $IMG 2>/dev/null) + if [ -z "$IMG_CREATED" ]; then + echo "Did not find image $IMG" + echo "found=false" >> $GITHUB_ENV + exit 0 + fi + + IMG_AGE=$(( ($(date +%s) - $(date -d "$IMG_CREATED" +%s)) / (60*60*24) )) + echo "Found image $IMG created $IMG_AGE days ago" + if [ "$IMG_AGE" -lt ${{ inputs.max_age }} ]; then + echo "Image is valid, skipping build" + echo "found=true" >> $GITHUB_ENV + else + echo "Image is too old, rebuilding" + echo "found=false" >> $GITHUB_ENV + fi + + #Remaining actions only run if we didn't find a valid image. + - name: Checkout repository + if: env.found != 'true' + uses: actions/checkout@v3 + + - name: Set up Docker Buildx + if: env.found != 'true' + uses: docker/setup-buildx-action@v2 + + - name: Log in to GHCR container registry + if: env.found != 'true' + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ inputs.token }} + + - name: Bake the bootstrap docker image + if: env.found != 'true' + uses: docker/bake-action@v5 + with: + files: .github/docker-compose.yml + targets: bootstrap + workdir: . + set: | + *.output=type=docker,name=bootstrap + *.args.OMPI_VERSION=${{ inputs.ompi_version }} + + - name: Bootstrap the environment Dockerfile + if: env.found != 'true' + shell: bash + run: docker run -v ${GITHUB_WORKSPACE}/.github:/configs bootstrap + + - name: Build the environment + if: env.found != 'true' + uses: docker/bake-action@v5 + with: + files: .github/docker-compose.yml + targets: env + workdir: . + pull: true + set: | + env.tags=ghcr.io/sandialabs/fenix/env:${{ inputs.ompi_version }} + env.output=type=registry,name=ghcr.io/sandialabs/fenix/env:${{ inputs.ompi_version }} diff --git a/.github/workflows/ci_checks.yaml b/.github/workflows/ci_checks.yaml index 14c376c..c6adc31 100644 --- a/.github/workflows/ci_checks.yaml +++ b/.github/workflows/ci_checks.yaml @@ -1,12 +1,7 @@ name: Build & Test on: - push: - pull_request_target: - types: - - opened - - synchronized - - edited + pull_request: jobs: test: @@ -19,54 +14,24 @@ jobs: - 5.0.3 steps: - - name: Checkout repository + - name: Checkout uses: actions/checkout@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Log in to GHCR container registry - uses: docker/login-action@v3 + - name: Build the environment image + uses: ./.github/workflows/build-env with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + ompi_version: ${{ matrix.ompi_version }} + token: ${{ secrets.GITHUB_TOKEN }} + max_age: 14 #days - - name: Bake the bootstrap docker image - uses: docker/bake-action@v5 - with: - files: .github/docker-compose.yml - targets: bootstrap - workdir: . - set: | - *.output=type=docker,name=bootstrap - *.cache-from=type=gha,scope=bootstrap/${{ matrix.ompi_version }} - *.cache-to=type=gha,mode=max,scope=bootstrap/${{ matrix.ompi_version }} - *.args.OMPI_VERSION=${{ matrix.ompi_version }} - - - name: Bootstrap the environment Dockerfile - run: docker run -v ${GITHUB_WORKSPACE}/.github:/configs bootstrap - - - name: Build the environment - uses: docker/bake-action@v5 - with: - files: .github/docker-compose.yml - targets: env - workdir: . - pull: true - set: | - *.cache-from=type=gha,scope=env/${{ matrix.ompi_version }} - *.cache-to=type=gha,mode=max,scope=env/${{ matrix.ompi_version }} - env.tags=ghcr.io/sandialabs/fenix/env:${{ matrix.ompi_version }} - env.output=type=registry,name=ghcr.io/sandialabs/fenix/env:${{ matrix.ompi_version }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 - name: Build Fenix uses: docker/bake-action@v5 with: - source: . files: .github/docker-compose.yml targets: fenix - workdir: . set: | *.output=type=docker,name=fenix *.args.OMPI_VERSION=${{ matrix.ompi_version }} From e0ce1f88391b3c8a1749fcce6635e4b4b5695f96 Mon Sep 17 00:00:00 2001 From: Matthew Whitlock Date: Thu, 3 Oct 2024 16:08:10 -0400 Subject: [PATCH 12/13] Initial doxygen setup Documentation Organize, setup versioned docs Deploy docs to Github pages Update doxygen header Use Doxygen 1.12.0 for github pages builds Improved Process Recovery docs writeup 'make doc' -> 'make docs' --- .github/scripts/build-gh-pages.sh | 54 +++ .github/workflows/docs.yml | 40 +++ CMakeLists.txt | 35 +- doc/CMakeLists.txt | 39 +++ doc/DoxygenLayout.xml | 265 +++++++++++++++ doc/fake_init.h | 4 + doc/html/CMakeLists.txt | 52 +++ doc/html/DoxygenStyle.css | 41 +++ doc/html/header.html | 81 +++++ doc/html/index.html.in | 12 + doc/html/version_select_handler.js | 19 ++ doc/html/version_selector.html.in | 3 + doc/images/fenix_process_flow.png | Bin 0 -> 67332 bytes doc/markdown/DataRecovery.md | 116 +++++++ doc/markdown/IMR.md | 6 + doc/markdown/Introduction.md | 51 +++ doc/markdown/ProcessRecovery.md | 116 +++++++ include/fenix.h | 530 +++++++++++++++++++++++++++-- 18 files changed, 1414 insertions(+), 50 deletions(-) create mode 100644 .github/scripts/build-gh-pages.sh create mode 100644 .github/workflows/docs.yml create mode 100644 doc/CMakeLists.txt create mode 100644 doc/DoxygenLayout.xml create mode 100644 doc/fake_init.h create mode 100644 doc/html/CMakeLists.txt create mode 100644 doc/html/DoxygenStyle.css create mode 100644 doc/html/header.html create mode 100644 doc/html/index.html.in create mode 100644 doc/html/version_select_handler.js create mode 100644 doc/html/version_selector.html.in create mode 100644 doc/images/fenix_process_flow.png create mode 100644 doc/markdown/DataRecovery.md create mode 100644 doc/markdown/IMR.md create mode 100644 doc/markdown/Introduction.md create mode 100644 doc/markdown/ProcessRecovery.md diff --git a/.github/scripts/build-gh-pages.sh b/.github/scripts/build-gh-pages.sh new file mode 100644 index 0000000..894d71d --- /dev/null +++ b/.github/scripts/build-gh-pages.sh @@ -0,0 +1,54 @@ +#!/bin/bash + + +set -e + +echo "Installing apt packages" +sudo apt-get update >/dev/null +sudo apt-get install -y wget git cmake graphviz >/dev/null + +echo "Installing Doxygen" +wget https://www.doxygen.nl/files/doxygen-1.12.0.linux.bin.tar.gz >/dev/null +tar -xzf doxygen-1.12.0.linux.bin.tar.gz >/dev/null +export PATH="$PWD/doxygen-1.12.0/bin:$PATH" + +#List of branches to build docs for +#TODO: Remove doxygen branch once tested +BRANCHES="doxygen master develop" + +build-docs() ( + git checkout $1 + + #The CMake Doxygen stuff is weird, and doesn't + #properly clean up and/or overwrite old outputs. + #So to make sure we get the correct doc configs, + #we need to delete everything + #We put the docs themselves into a hidden directory + #so they don't get included in this glob + rm -rf ./* + + cmake ../ -DBUILD_DOCS=ON -DDOCS_ONLY=ON \ + -DFENIX_DOCS_MAN=OFF -DFENIX_BRANCH=$1 \ + -DFENIX_DOCS_OUTPUT=$PWD/.docs + make docs +) + +git clone https://www.github.com/sandialabs/Fenix.git +mkdir Fenix/build +cd Fenix/build + +for branch in $BRANCHES; do + echo "Building docs for $branch" + + #TODO: Fail if any branch fails to build, + # once the develop and master branches have doxygen + # merged in + build-docs $branch || true + + echo + echo +done + +if [ -n "$GITHUB_ENV" ]; then + echo "DOCS_DIR=$PWD/.docs" >> $GITHUB_ENV +fi diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..1f6609f --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,40 @@ +name: Publish GH Pages + +on: + push: + branches: + - master + - develop + - doxygen # TODO: Remove after testing + +#Only one of this workflow runs at a time +concurrency: + group: docs + cancel-in-progress: true + +jobs: + build-pages: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Build pages + run: /bin/bash .github/scripts/build-gh-pages.sh + + - name: Upload documentation artifact + uses: actions/upload-pages-artifact@v3 + with: + path: ${{ env.DOCS_DIR }} + + deploy-docs: + needs: build-pages + runs-on: ubuntu-latest + permissions: + pages: write + id-token: write + + steps: + - name: Deploy documentation to GH Pages + uses: actions/deploy-pages@v4 + diff --git a/CMakeLists.txt b/CMakeLists.txt index 170b576..ecaac8b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,8 +16,9 @@ set(FENIX_VERSION_MAJOR 1) set(FENIX_VERSION_MINOR 0) option(BUILD_EXAMPLES "Builds example programs from the examples directory" OFF) -option(BUILD_TESTING "Builds tests and test modes of files" ON) - +option(BUILD_TESTING "Builds tests and test modes of files" ON) +option(BUILD_DOCS "Builds documentation if is doxygen found" ON) +option(DOCS_ONLY "Only build documentation" OFF) #Solves an issue with some system environments putting their MPI headers before #the headers CMake includes. Forces non-system MPI headers when incorrect headers @@ -25,28 +26,32 @@ option(BUILD_TESTING "Builds tests and test modes of files" ON) option(FENIX_SYSTEM_INC_FIX "Attempts to force overriding any system MPI headers" ON) option(FENIX_PROPAGATE_INC_FIX "Attempt overriding system MPI headers in linking projects" ON) -find_package(MPI REQUIRED) -if(${FENIX_SYSTEM_INC_FIX}) - include(cmake/systemMPIOverride.cmake) -endif() +if(NOT DOCS_ONLY) + find_package(MPI REQUIRED) + if(${FENIX_SYSTEM_INC_FIX}) + include(cmake/systemMPIOverride.cmake) + endif() -add_subdirectory(src) + add_subdirectory(src) + include(CTest) + list(APPEND MPIEXEC_PREFLAGS "--with-ft;mpi") -include(CTest) -list(APPEND MPIEXEC_PREFLAGS "--with-ft;mpi") + if(BUILD_EXAMPLES) + add_subdirectory(examples) + endif() -if(BUILD_EXAMPLES) - add_subdirectory(examples) -endif() + if(BUILD_TESTING) + add_subdirectory(test) + endif() - -if(BUILD_TESTING) - add_subdirectory(test) endif() +if(BUILD_DOCS) + add_subdirectory(doc) +endif() configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/include/fenix-config.h.in diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt new file mode 100644 index 0000000..10c780f --- /dev/null +++ b/doc/CMakeLists.txt @@ -0,0 +1,39 @@ +find_package(Doxygen) + +set(FENIX_DOCS_OUTPUT ${CMAKE_CURRENT_BINARY_DIR} CACHE PATH "Documentation output directory") +set(FENIX_DOCS_MAN "YES" CACHE BOOL "Option to disable man page generation for CI builds") +set(FENIX_BRANCH "local" CACHE BOOL "Git branch being documented, or local if not building for Github Pages") + +if(NOT DOXYGEN_FOUND) + message(STATUS "Doxygen not found, `make docs` disabled") + return() +endif() + +list(APPEND DOXYGEN_EXAMPLE_PATH markdown) +list(APPEND DOXYGEN_IMAGE_PATH images) + +set(DOXYGEN_USE_MDFILE_AS_MAINPAGE markdown/Introduction.md) +set(DOXYGEN_LAYOUT_FILE DoxygenLayout.xml) +set(DOXYGEN_OUTPUT_DIRECTORY ${FENIX_DOCS_OUTPUT}) + +set(DOXYGEN_GENERATE_MAN ${FENIX_DOCS_MAN}) + +set(DOXYGEN_QUIET YES) +set(DOXYGEN_WARN_IF_UNDOCUMENTED NO) +set(DOXYGEN_WARN_IF_DOC_ERROR YES) +set(DOXYGEN_WARN_NO_PARAMDOC YES) +set(DOXYGEN_SHOW_INCLUDE_FILES NO) +set(DOXYGEN_WARN_IF_UNDOC_ENUM_VAL NO) + +list(APPEND DOXYGEN_ALIASES "returnstatus=@return FENIX_SUCCESS if successful, any [return code](@ref ReturnCodes) otherwise.") +list(APPEND DOXYGEN_ALIASES "unimplemented=@qualifier UNIMPLEMENTED @brief @htmlonly @endhtmlonly UNIMPLEMENTED @htmlonly @endhtmlonly") + +add_subdirectory(html) + +doxygen_add_docs(docs + markdown/Introduction.md fake_init.h ../include ../src + ALL + COMMENT "Generate Fenix documentation") +message(STATUS "Run `make docs` to build documentation") + +install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/man DESTINATION ${CMAKE_INSTALL_PREFIX}) diff --git a/doc/DoxygenLayout.xml b/doc/DoxygenLayout.xml new file mode 100644 index 0000000..d636ef1 --- /dev/null +++ b/doc/DoxygenLayout.xml @@ -0,0 +1,265 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/fake_init.h b/doc/fake_init.h new file mode 100644 index 0000000..a9afa16 --- /dev/null +++ b/doc/fake_init.h @@ -0,0 +1,4 @@ +//!@weakgroup ProcessRecovery +//!@{ +void Fenix_Init(int* role, MPI_Comm comm, MPI_Comm* newcomm, int** argc, char*** argv, int spare_ranks, int spawn, MPI_Info info, int* error); +//!@} diff --git a/doc/html/CMakeLists.txt b/doc/html/CMakeLists.txt new file mode 100644 index 0000000..70677f8 --- /dev/null +++ b/doc/html/CMakeLists.txt @@ -0,0 +1,52 @@ +set(DOXYGEN_GENERATE_HTML YES PARENT_SCOPE) + +set(DOXYGEN_TOC_INCLUDE_HEADINGS 0 PARENT_SCOPE) +set(DOXYGEN_DISABLE_INDEX YES PARENT_SCOPE) +set(DOXYGEN_GENERATE_TREEVIEW YES PARENT_SCOPE) +set(DOXYGEN_FULL_SIDEBAR NO PARENT_SCOPE) + +file(GLOB CSS_FILES ./*.css) +set(DOXYGEN_HTML_EXTRA_STYLESHEET ${CSS_FILES} PARENT_SCOPE) +set(DOXYGEN_HTML_HEADER ${CMAKE_CURRENT_SOURCE_DIR}/header.html PARENT_SCOPE) + +if(NOT FENIX_BRANCH STREQUAL "local") + message(STATUS "Building documentation for branch ${FENIX_BRANCH}") + set(DOXYGEN_HTML_OUTPUT ${FENIX_BRANCH} PARENT_SCOPE) + set(DOXYGEN_PROJECT_NUMBER "@${FENIX_BRANCH}" PARENT_SCOPE) +endif() + + + +file(GLOB DOC_INDEXES RELATIVE ${DOXYGEN_OUTPUT_DIRECTORY} CONFIGURE_DEPENDS ${DOXYGEN_OUTPUT_DIRECTORY}/*/index.html) +foreach(DOC_INDEX ${DOC_INDEXES}) + string(REGEX REPLACE "/index.html" "" DOC_VERSION ${DOC_INDEX}) + list(APPEND DOC_VERSIONS ${DOC_VERSION}) +endforeach() +if("html" IN_LIST DOC_VERSIONS) + list(REMOVE_ITEM DOC_VERSIONS "html") +endif() + +message(STATUS "Existing documentation versions: ${FENIX_DOC_VERSIONS}") + +list(APPEND DOC_VERSIONS ${DOXYGEN_HTML_OUTPUT}) +list(REMOVE_DUPLICATES DOC_VERSIONS) +list(SORT DOC_VERSIONS) +if("main" IN_LIST DOC_VERSIONS) + list(REMOVE_ITEM DOC_VERSIONS "main") + list(PREPEND DOC_VERSIONS "main") +endif() + +set(DOC_DEFAULT_VERSION "develop") +if(NOT DOC_DEFAULT_VERSION IN_LIST DOC_VERSIONS) + set(DOC_DEFAULT_VERSION ${FENIX_BRANCH}) +endif() +list(REMOVE_ITEM DOC_VERSIONS ${DOC_DEFAULT_VERSION}) +list(PREPEND DOC_VERSIONS ${DOC_DEFAULT_VERSION}) + +foreach(DOC_VERSION ${DOC_VERSIONS}) + set(DOC_VERSION_SELECT "${DOC_VERSION_SELECT} ") +endforeach() + +configure_file(index.html.in ${DOXYGEN_OUTPUT_DIRECTORY}/index.html) +configure_file(version_selector.html.in ${DOXYGEN_OUTPUT_DIRECTORY}/version_selector.html) +configure_file(version_select_handler.js ${DOXYGEN_OUTPUT_DIRECTORY}/version_select_handler.js COPYONLY) diff --git a/doc/html/DoxygenStyle.css b/doc/html/DoxygenStyle.css new file mode 100644 index 0000000..770b1c6 --- /dev/null +++ b/doc/html/DoxygenStyle.css @@ -0,0 +1,41 @@ +/*Move qualifiers (e.g. collective, unimplemented) to being above function name instead of bottom right*/ +/* It's too easy to miss as-is, especially the unimplemented tag.*/ +table.mlabels { + direction: rtl; + writing-mode: vertical-rl; +} +/*Undo the weird writing-mode changes at each mlabels table member*/ +table.mlabels td.mlabels-right { + writing-mode: horizontal-tb; + text-align: left; + width: auto; +} +table.mlabels td.mlabels-left { + writing-mode: horizontal-tb; + text-align: left; + width: auto; +} +/*Undo the table direction change in the subtable of function parameters*/ +table.mlabels table.memname { + float: left; + direction: ltr; +} + +/*Make the qualifier labels slightly larger, and bold.*/ +table.mlabels td.mlabels-right span.mlabel { + font-weight: bold; + font-size: 12px; +} + + +/* + * Hide the "UNIMPLEMENTED" tag within the function's detailed description + * It's visible already. +*/ +div.memdoc span.mlabel { + display: none; +} + +table.params { + word-wrap: break-all; +} diff --git a/doc/html/header.html b/doc/html/header.html new file mode 100644 index 0000000..edf8ba4 --- /dev/null +++ b/doc/html/header.html @@ -0,0 +1,81 @@ + + + + + + + + +$projectname: $title +$title + + + + + + + + + + + + + + +$treeview +$search +$mathjax +$darkmode + +$extrastylesheet + + + + + +
+ + + +
+ + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
$projectname $projectnumber +
+
$projectbrief
+ 		+
$projectbrief
+ 		$searchbox
$searchbox
+
+ + diff --git a/doc/html/index.html.in b/doc/html/index.html.in new file mode 100644 index 0000000..e10ca8c --- /dev/null +++ b/doc/html/index.html.in @@ -0,0 +1,12 @@ + + + + + + + Redirecting... + + +

If you are not redirected automatically, click here.

+ + diff --git a/doc/html/version_select_handler.js b/doc/html/version_select_handler.js new file mode 100644 index 0000000..159d913 --- /dev/null +++ b/doc/html/version_select_handler.js @@ -0,0 +1,19 @@ +$(function () { + var window_location = window.location.pathname.split('/'); + var current_page = window_location.pop(); + var current_version = window_location.pop(); + var base_path = window_location.join('/'); + $.get(base_path + '/version_selector.html', function (data) { + // Inject version selector HTML into the page + $('#projectnumber').html(data); + + // Event listener to handle version selection + document.getElementById('versionSelector').addEventListener('change', function () { + var selectedVersion = this.value; + window.location.href = base_path + '/' + selectedVersion + '/' + current_page + window.location.hash; + }); + + // Set the selected option based on the current version + $('#versionSelector').val(current_version); + }); +}); diff --git a/doc/html/version_selector.html.in b/doc/html/version_selector.html.in new file mode 100644 index 0000000..ca92356 --- /dev/null +++ b/doc/html/version_selector.html.in @@ -0,0 +1,3 @@ +
diff --git a/doc/images/fenix_process_flow.png b/doc/images/fenix_process_flow.png new file mode 100644 index 0000000000000000000000000000000000000000..e94029a9dbfacc7ed40755d1b1c387165200b7d4 GIT binary patch literal 67332 zcmdSBbySq!`!)(V;LyWR0*Z8pgmg1>Nh2MCG()FI3J54MG@>*}2uOFQNP`m6Dj*05 zg5=rb=Ns>P&ih;I{B_nkf4Eq}Jp0-E-h1D1UDrJ^4>Xks@u=}IFfa&Jlo2`@7+9*{ zUndj?{Kgq-#tMF5dg>_2VN`y+y@i26hoORy)$=p|VTRkNsZVMCHTuK*gb!(XPS4pQ z`B8F$^x8~?DhRpEk_|V0a=bz&y+N4m@U0mB$37|kw7;Z(Rz49w$@*A3=KIX8;#ZOkI)4zumhr!qh{rdw(g~_7Y z%KzUFg^)ytVURMLQ~&of;G^1W_y6_4-yb2OkWYD!hH4V+YQ__12RFX^d%M7k^U@yP$ zW2vLK z5;YZ-TrH_T5no5Mlsc}j#P5h8LwHOZWtu!T^Z0Ft$2EkPn4ou*Sz?7@`wZIMX(7BF z7di>`swo`)H%cJ4u)!i9@N4m98WL*EwQd-(}>Zhzfsve!tT-`dU1$b10d0E6c5& zq-_z|L#S77#1|Df?bdG55qvbM#yj!&^%KRDMbDSP*gUxqumn}h4aHlbr;KlyRl{)z z^L%#aBMH}jc3iU<39R4+{%kw>Ui>;3Zkv>x9I2*KXgj2lM@INW*vySi5locafC;K> zI`HDoR;r0>F6X;eoeP(fy``helVyCTsn<`h*spLHVmtUAKED>&Jf^MAvHOtTg<${q z+&Yi!ac7gbq6|5fah{;_r+eFzg|V(vFC`A4@znQKQCSlH{B`ydat$vgz_v^>VDF57 zs8 zwH7OU(hp^m@5IxdfB#tPE&gf@YtEbN%b!oJ)|>ne@~V*P*&>OvA0RyKwW zr}1j}e(`7Z-knc@ose-PvE|-gLk6F&-dy!CZVwW@Bjy!-_;plG_=_Lo@!nDvhhb%Q zk=R+OaoyWfn#DV#nfGI(N!R#i-&ttwE;MK{)3e#%Pp6@H6?)_5r0?<*RxUR>{#yg< zFQs|PWevic(zqVVd2dgXc$S#82wk48zeMrb^rtu9+^b@_y!bW7V1P@}kuB<3R2m-Q z_1zgvG~yO@W7Ofs5aYh{=Ne|oqj7nZsOMKf?|VuPUB=bc*!ty0+8?N@d0KpS^A?*t zr;g_=v~J)KD9_-RTe`I#;8?}qkq*vQMX|}AEcw13H!D)ln5^YJCTeZJ{B4qZGG`vB zF&(K-h&eGnHl-!4f#l5*ahD5vapk+WsFG&plhFy~VKx%0!xG-m)skesif~8N)X;yy z=krSd6OV2$K=GJm*1Ilod_DZ}ehg1Ce{`nIFxK1bjSS{niw>#d-G#K_RPL-IH%30y zbjG7<%Py?>j;9@+w5Q$NzrW|d)Jf%f*zNVgq-UQQd-fx^n-dmKFPVo|-e4lX&sJoe zR**OCF>=#QkEQWiY%3YeM*CD6F^YO@yxCi7A)V#BH(z6`=6D`>vPbTjBN-^Q@Lk-&kjv+DOmO1IXgE!2t!fz5&ur5_SOqV>yQCmW>>mpj9o!Bgj5E}TP zLV=8#^C7hN=0pe?yAA@zo?u~AKnHe#do)`#8IO`Dd;V3bvZd$Faz{rx=}}lfbbi>% zHJt;U*!Z33rOh-89%v+40xW@~!qVpH_2r3z%vrLQ6gyVDY;=`Bt*8f69a}W~4Cm{( z@Y#(&qED|{*^=)&8B9n=*ORp%RNw?hv7>h%;qKfG=>PO+xszl=CC5YlIsk#YxA*P! zQ}c3nsPgUE_zP(-%vM>!f)IjrnLa~{C(KunY?;H4Ig(k=^_#u6AQ?AS6Fk7nV|nl1 zVuXy*w)@HKbE(R?c*loq43XyE5-6XdlZC=Rh+!Q!yj zJJaIcKuHVV*e88UL_&?|Rp-;LJ;x-K{64E;`paDq2&pmzNuUtdQ{Q%n6wnLpfQ#o@c@j0E1O z3Heg?<129`M#VVTBjPO4_S`IbRXW-v$aIB?p%KjwJfXK7`fsC0p)~hQ15TwOmsS^@ zcbRs+X1P)DY3j&$-J^wuCCS$I$J4NSLGv&qd(JR1Ny2k1{8q86wZmIG8L;t(*7{;` zvrb>!5!d0?E4eY!BmT3Mk&Ay^97@9FQAI}IN!yFzM1CEm0H=|Hp531_W$Y(PV~VDl zg?^#<+p(Z(`xnyVyFRd+tDdX!5Hlle}?C zSkSGx4d$Z|}A61S#`Usmm}K1{T5toj=KXOq(Eez=0VaVVc+f?MP718j&B zrhB3huPls3Rd$?VjGmqZ+!ok5f54G_DCHt2Qe3&K!L-qhZsM>%WSnjk!#rouq=1kv z4S{?7vvHlsy@p56GQP72+Vw0fji2cY0(S#Dt9UW4+949J9Q$3zRrBe4s=T9}cO7_e z49wM#tdUCPIfimUy7W&ddF)3DA8l5F`>82ET)PE&a5e&JC8p3=0FBQS4xW!yff@rA7_4(f{HjG944b zq8CD{Oef?Ur=iM~r<=H5~vfrvt0w2$OwWF}A00MXRw(}bpsuCv>VE@MgrmoFmLC(<0m~bK!SFvCELDOgQaZPpH+W!V8>g%AP$&FUxr|g9HB>^>&AprlvX!hx?Qg^rJ}EmeT8?((Tpy0`;y0+XY0KSd zypju_UDZk6RoJ=9)c$xZC$BP&Mn?rjE#%C$7RfLKG8zW1r!Xpuh^I=7s*Fl6$Xvys zTx`ZC>CY)s2xN$#5dE{@u3z%c2x3IY{Tne0sLtfF^J_y&(_0J|Eb}(>NB0|VVcvm! zfHP;4$(P3n(dYlb%Ayit-&M3iV1An3=r4)amJ4wg>=Jf1s;_lT^)o?j;NNvc^s+0E zQmtD*W20|0u#QWoY$~Ph640TtWVS)9z(Y+0GDBb=UWps_CUicBPQVN8*MY0dCLDZE zFip5CkHZ=MK*c(ge=;w^(z4f60ZvsGMrf2{J(YUn00;ACZc^9gscDJ=uDi~u2TWzf zJcWmc5uw8J1YuGY+=XYSxN?RdxUDJIJyS$jGFNZ&jIGp^QKXb(y+rO?qzGH%#-lgF zQa82b=%mX_;%S75#7iA;Xx^pT{(HHk2wWOfCQ4!<7k?#kq5Q`={w&sI(ESwNpbMfp zAq5&dAshcSML2U%PrvZ4eONqgSDuK+3%A__;ICCekX*E~B@+nyCfy)h63KGG0eFkc%6{4^lIR!ZLcbz~T4HM0mY<)E!Rr1JONPRNd zx+HOgGw|}5%)IUKXB)=72&#|p@ENURR)!;1Ana@YAr-w)1SQ2o0ffpX^~A`~Ghgg- zM`-r|yFx|hlb9QKt0q^MWl4bFn_wDaT!r0(=R+QBXnVIdBCVN|$}pA-tf<0lahF-D zb2G?QP_C1u`TO*| z)>-2aLK2%8mQ#b5x7b}eWLZ^_@xy+v$0bzx(@mJQCyh}dF7}mW_RX?a^o=ts?TFiS zT&DcrEldWU?`VK2Gpf#4cOSnv#__mmSg&43vU@Ko<@jf$m^!NU4L0o!MoUc0d;~79 zeFEHhQN;U3vh<6KD7+*%0}&SMast74xpH2_DRvO!i|L#5)3l!z$Auvf%^ZT~o90RG zdOk65tO<8#6JAB!+517^Lh`74ceB4HQxNp_cQV@{R+__@F$c;wH~Z+G@e@yhvWOE7 zS0rr|uo|7}-#E=+b9gz`!+dLKGA7xP|3pV3I!Q)4I)ks+qA40<-ghHsId8|LIw8t@+0>E+~o*)tup)d+j-cRVNCQe>MTC20NXFjY-oq}4u`c?4!bG23RA}N zt&%z+#DK;Ke+4VffRctqNHf|jFCtc<+Fus7OwRriewL6iG74v|<$T$?zKpEKuyFbM zm|G|bGPDr^<>`0#XFq%)WwZk$xV^;^xVy!j&4K#dcBzi44S0@Pz|Rs|8?4G%S3r%w ztX&Kp41+|$Zx%%QovudprORaUDMGr|z33r4oY~Yfcsit=!x4?r5;t>8Z|`p^m=lJX z+9%z9YtA?k4Nsb4h{xixpN}ZY-MLjj`!mb_02v(gP(b@If7b6q97ZCPzaRfr-U&C% zw#uJD39tQ_o)W%jKL4S~uZfZ4?Y}tDoZ5{_zY_e&MWg4_eonn!(FbfGw_P+VZI|aM*c{g z9KvbYcGe)PkaMa6>YFznQI)g8Y=cxV-J_fgbKt!K`Hu03z%;t02(Lo=39fcFhxI$v zsRztAn`W#@0;JnpGuZxUJ4x6nc4BE`+4D~@ToE!v#4B<(X52QQ4Z?pGf><%(65@hE zp_NZ%Cgn3Ym1l33Vr5|ZhkN5v(4qQDX&V{#86sCAh}S6PYy@E=;h*T8AvJC&g$_DB zBz_po@;w3b4FjG&1tL~R>IhT#C+G3I(NygdJ(Cdljl3Hk;mH98kmc^`QcKc!=I=Dc zV@Mqyoghjf4oeR=Yt^yCi*Ox5A|q^ESqTL=mW!~&yA zQJyqNjN}9NKyKa2kavae9L;Q^`s8!}m6aRiYRUn4J7hVSFzyUFOsY?bBxWpI5y9?< zBv?ri-p}74Rp#}|6OAmIk^c7%?Nu;KZ#0D~7`5NsFJBYw6QCxu$|EDwIFx`BVZNte z#)ufDPQ^7+sT%6G#oT#jus=$UfG!)6I7JXbSp!%0FcUFxpKr*4`onzwbQYH(;%)+= z>C;f5giGq;nyLc@HJzxq<*Kip3ehWO)3O)=@66i}tuXWUs-hL9yX`Y-)pDo@lm@uY zen{pJChK3_Mfirdg_u9P)|PR^@rXG`RKZuh;HI1&x}nE}XEIk*L27BfH409YSsd0M zEN6No9miQ~+{3fGjj;{wzh4>$v#NGtr7}RYd+>0}s0PKkM3HXYk&$&Vi)OhS%PM&h z?yLCNXT8U~<9y}ML_|U#W6q|G7nD`SSv_I=NYE5*>EHG1W?uOHn_>7)TssJ7>+xPtTUvP27 z7QEsA%gOCAjOWVV9DXe=O-L*8X(<%>GcH64J$>!I{i?q!kG_euHCafyWZLY-PP^oO zH!kES4E$X@Sb0syJEk%w9rA|yuV-tq8zYqTXgVF36d=<1Dx-vTSgEf2~O z`)dsVxf0&Pss^v<<%~evA`}8FsDs7%y4V#8KKaaYP42HfAf(`bUDKRns&~Lh(^f;= zeCW+t5eF9fqiHL#BEr}SptulE-@hOC!^KoV>FDT`SayY(Q&Yc;B-w|6=Ol7*Xd!=J zyhleDGg!%%7j%Hvxh`dbS{;r2l9C(OJFEO`Kj$ImF*WMIuGUUTeQ{jyZ!@~78VUH}Qx?R!p6C80={3k-`I z-~KiPPO=OGeU;-3N2c^ssnHC9D7?xNuH-G-?IdifHstx2!nKOV~FD88h9g@-?w zbfLmj?pGNX0(KGp4*TzWgPwK%`SopUV<>s;+jOaJX@^#}i0r`2yOh$0%L@%|tT3hv z`$7OwAEALS0N8cgNn3Ve$QDuXnrEL;g|)r|sFiHVa0(|Ymxe+tMIk`^CIIAbxw1WnfIOSkGE&m8d8H+mJh#y8o0fG`=j{J0=v;n zZGcRbsAupudtwgB1v4AHPUSIE1!&n!#d;kUorf_$>$L?ah*dQ;H7^h58~r7|$bviO z>(#|*1x}g>eU<$LgIwH404!pQ9@1bAr}Ht|j%KFtwEU=v?odnRQaNSU21zCN(;wX} zZp-a@J7ipXPcM%RIbx=h)Qnm$Sm-p6@KudVeTJMg(75D<}98T65SCi=-(m(Cf;lL zDRrO=ka|Q&$hTWJLTv_KOlV18GA+j1^vC7i5%-CnY&_os1}_5q179#MIPQfI11YUn zB+3wo;)~lVuf%;Rbg|jxFa)4FAG1YCte=K1Dr0eq@)A@q=y=R0%n?zSBcbLL6tuyk?^ zCHaAeR3%>hilYjRkg=R9($FhZB`3+^w;!hq*lYR*xBeobQU~?gnT`MQ^!mvJ+g>-P zNqzY|QRDqFaKDFsajaqTgUd+{L6QZ}*E)Fp8Wo;8S$^tPh+EkMg+f*WvP9hT2F)rV}Qrme6t(&lPG4Ho@Dk6mb`zx|H16Ekt z&@O=2DQW=i0#acz2HUqqJ))ZjbzVsZ`l#(9nH6Oi62yI7#XL5k6{bx(Or?)hZiDO0 z{G!soC{CuEuF7>u%)OiN4|3m)La~zv3(c{pB(qFJlQ4cyRHCh3xIK450`@SdfQyy` z_@P_!D*)<$b9THlg{D?`LT#-eKNv#Jf1aKEld%N%iOguGS%?pW zG})g41L`PKLrh{c>!k-Qs&YSXk(RWu*X#y3ScWGOo{Dr;=zT4BG|Re0$~^bRx6KAT zSQc^^x6y< zOedbLVF1DIJ_n}XJ0m%kO#BGBg;psl)5M0en@(B4=4|L?|2XDd}iE!gCt8l6CBS=-Ke z{1FPF0MifEuW%x@$=jels`UMU81g3H#{gbi=02F59^>#XpSJ4aeGpXKy~hL|L$G4O z^gG|q*2VB?dk_Z9iQDCEG~@YUfYbkm8VvT9Wb-$$`W`jdug2zY83I_LAfW45>hH|= zf;0iAZRd`Fwuc$E1O{-X58}Z=n~C-a1XlMY1Xsdz>DLFU&`z+6>{V~QU97<_g?~AI zk-^qfiA9ShBumhVg(IR#61}}R|K8pZUYOrX=uP|X2we9&JAl<(o4@NzVFB^@`auQo zSg|I>fWgTJSQLxvrMr?NdiyA}T+vAf8G7^i-Ynkc(_T}s!~qMi8(*tIUu9nilQy-m z>+RKLP26&20H{)qzE>DxN3~5h)SU zK3Kd_dM$!ugIi`d`K`7`mvzpod!)=`Z~K>T>`a%EJ~^6D(rfW?$>g{FU_+enZ!G{G zIoksnb!cj#iCbGrz4K=|08-`_=L9^DN$Sua&3}weGwE2cICXude&Ijv<T+RjLHL zE^#=?@c$-?aUmTfei!GZwX5T=o0Ip^!xfYX@mNxX0)}uL?Us&r=8FA)eN`wYN~(2R zmICP;)l7*tqXreYQzYLG|C-Q`29wc%d6yvRkbbjwJ`p^208%kj^Im3B3AAso(YPe zEDryl!8sv6j8bYzE4aAS59q=5a^|VdS9*zeG6~2)P04_xH{WssUwC}`=<;6we5Q^G z7nk;D^A1n*uaDxp!p?871t^#cNN(MC)XOwjbsa&mlNOF8gqoZEP79@NQs#nC$K6Kw zI~o+0(K?bE{1~s9IU4vr)^qi7Glp7|dI*2y2K3R#gwUdf@3(}KT$G5lzkRlNcG;%) zQ9>|JyR521>&oRmp+$$>#pQ9+f{yf9!ZH6_{@SNTe`1Tr#xq%au3hUS-myNW(3OCa zk}_vC%Lk2?>K5Byo*fHKQvKNHK|nKWp|VICe$X~q8d zbU@bY*hL-_vNu2*$Yni8msZsoYDz>aA%dqpto)8=N?Mb8tm#iRFN^B6!zcmYM*JvE z*{hf&;iHvfZ4s4pEG5+1on*%FrV1_w{mRPmc8j0`1!@uGn?{Ze!1vel3z~T61RbiJ zEni$AWwYD zO~dz1QXtRSnn;p$YaXc#gCu{Oom=4f!}oUa8EemmbHsgLpzEP5VYecH$m;;z3gL|I zLv)-w>byY=GeyQyaN{{qSKfcLEfsWn?-Gz8UZeNt3fMibF8qhHMFngJ(-X}f4pT6< zG+X5Xx3x#@@m`9QS$N{aV!07sC||g(sx^T9jID_(dA+bu&~;J7O67UVkBO}VS}`x> zM@u*k=fF9Xnl^bjG$Rsx0V#$qG-@5DimFyGlu-N=oNA2A#2ZJq!=}NofDketQ%7Bt zx1GkfIqzLeOQC%;rr)0}t+jiffEl$11-mQ!2zgtn>LZ zK|FSL_HJ&d4kZVuZ5`ud;H1oToE+^HqOvHQ5A%qJ{Ex^;tEsc^JP$M97anL8eeViu zy|yge_Ii?7YiL{ggBykz+D*Tmf>*sTn__6rzNdAAgx#p>&C*w{Ii-+u?uJD!*lY-%OF$~6T71w`BX$79B{;fc?q9eVoOKEHn~th4BAYdI0FSMs3C z*XA3hl2XXAqnmfo@6V6DfQDw?DZ1Rw5rL_ms zO7uVmzb#&cd0Qru?+0*keK-V8s-TMN3y(RaFNQ#H5xAGTpWhFZx)LUsL0QKS>_Y|;+E_)Wn#`*8o70qUfSdQWd3%sQ-M6%oWa~ELxgbDMV{41~{784jqugmj zjw(YxRkJk7CSb{<7GDg7Izf3WKz)j*ZgpQafxldAFdrRRK|D;WDxn*IL-@p^8EcY_ z=_VP1zP<7-0nz0M5WLSRLoEQm)3^+#v;IkN#i!+X)`T6@@N0$se>T&^XVYj1dn9$M ztraGY-8wG&_k@K}D>ARk4Flh-_3i@kh{EgOGxnC{AF>Y3^(kuZYF|Id-{H{DpI6BC ztyaiN_{pv6Il2U(XtY1b-mj4xtDxCKNVd~JErsb8Kt)-TR2Y2>nY)(uh468k# zRGU}W;5X3l>^@rwVD{;FI56ovn!?_)^(V5Qx0m}B=iNGu=8yVeOy9mcq4y#ihmd+4 zL;@`+Q7dq`UUMHoHH3)M1d;{^kRmj1j*`i!hTIK4mFbCR6s#IvI<95DE3k#h*UB zSegT37(UT^>)rZZpt2iIpbzJ{5;@BOAdQu`&K;y#nE^M5(izcOa@k21%z*Z&yc456 zzUXFr5|_=`vV3udxDa-oEt9;X+K}ZOx#aeHU(h&EVVGyT4Qcmb&74U&w2IB2p5>;} zCz9ldmM?L5n!)rFdE5Cv_224|WK>!ylo(WGPOSP>-d;;u8AkD$YZ5%z@k!pjIdoa% zJvR8K`g}Adg=|^q;k(&`+2*B3oIir!zbMYgH2W5;kVSkq+6bFZnOHRY=FkTt#+2ha z1@vk_VGoz>v@4e{R84_Xlz>1sKVy-3HW)&Pg)_+#?w8^NEAC&DMZuh3PCYlqO*+@u z$w~BlbYhZCQ5C*aWnK|p=G`)6@afX-j#mEt{D;X2DlINL{4*YO$@LS}RNUYGZq{uk z9RytA!ZdKVuDP-2G7*Jh8h+cpQ-O$q*Lr@_oUeDGG4r0$Z?-vb#4L`{DOu21gOQz> zF#Th=*5>Wwfxdv50XBn}2Mt+VBYVPujoUMOAklS^^te}-{u-KeyY}5*ClT=bhYDIs z)cd$Uj;hqq8(}N~mFfUuKy2CVnKH_u0`~y?(eK|56)@CPa^<#NHHGFo-g+pPCUBbw ztp>|KbDKE?GLea!cY|QnpAXgtrhvPC*F4f#PdOyXc(Ze_q4BMp!{c(p9i*a1%g`)( z-G*(MBf37?ubG6? zJv}zX7KC6FNyv;mU0!Usp1oES-c2^Ax=Kg)A}$s=EfR?mP`^*At(X~ zUcR~Z)a&(7r57yO-8~(b9GdXLCeLp=APdTu3r~&};U}2fQHt5jN&IDXl65UO5>QwC z-riOCGY4=8t2Cv*+mi?E2dWv{qh%+v+TL-inm-=`krTh|;&6n4Qj4h@5;k3=K@+eV zNf}W$gqxVr?!Hz28hcL&G8kTdi~hCUSL%^nmW6}&=iI4*iXUGtx!D=S2@ltpSM1QI zE@;#b?+NdmvdC$m?bW}rk_itwm(PX?`8%HixRG>E6oN0#u8umI;5M#h>oTKOGp=(~ z0GZpQ+lO#w>!!yW$j_HwW&5@WVvZfn9#;dy=%WoIOyYQ!`S&^*Ca=(k_YFpORzw7T zNg!_qIF;htTRc4ob|Jk0SK)^X@AiZl_ozgbd^)It%T0L)!TPZz!4-W4t|*eo!{wii z49T7@f0zuQKu7ymbOOR*ts`l?ZX7CwiT&YnD9f0U9LbidetKo|lw0mCL+uyegH0Nd zR=4y(cp`l&U;q2Tf#+`@eWz~XU2tif*x^3J(iit!78?^4?Rle~WN{#>Ih{7pG7`3{G&3wkip28wdy zy5|O!rYb{2L$fq%nP^OhIr*)5gX^7DrQx=dFKVEOF9Pby?RrQ9jSeK}@_4RW-*;n> zWJ$~p53;$r1<`(fGzzC&1FA&hjqK-Mu_ESq!IGCOa~cn3BQn0xk)Zd>DI+Gy`v<&^ z1q`0~bGbvB@YcP~M-}^P|AGVheZ*yJX*(xV8I_2@Win?gegrByJ&=lZZw&#Vr}#{K z+3w-9ZH|i<-X8<%h`33acRn#YYOtuX+$VN|Ya+UwV_!>OUrg!YL&+3r@1730Q)?E2 z=?x)Nmfk{#Gap1L-!n#sQO5r~DRloOCBy$`L$~b zuA)=IYFMV};c=-(?`Dq&5kq1=F~-TD9vC%ztEaN~OUSAllEwY`9W5%Ym`N4XEo$|; z-Ic<{37bBz6zSN;nx{k*9AC#I(UhFGXtAjBeE*47CUq?W98`Zk@PTk@fsKAJl9W@RZ8Y@1*oYUDT&|6)IG}1u zvd4A5FY*~@1kR6S%vim;JdZ3Rd#BY{LSmRvf_l>`k-=7KLN5D|bQ^}N zs0KV`z@xlJcUlk?VBDgwi5Y=wy)k+qRGtAmzXVjyc?zwAnZj;7{ORvKy7Yfniu&=CyvFaY@5711^1}p>HI14tmvp z_aTM|r!SXSK~Z!zlEfT~td7L7QxjeQx%uQ#h{+<51uF(@rWls^{`?{osAcl+StFHcZL_t&sEG{nZ{- zzDc3fXaw$o#p|F$T<@LPQ9jcqt8W0Fdvs@A8Lcyp!8$VT<)j&UZ1g1@+Qx&vW1WMTw%>hv zQB9`;(y9j5%0+GAkzOZua8+nBlM!2JMS=hBzWNIFTiI}&m}Raw88!fYI#=9K&EFcn)4WM^`wdVHk2kun_bxv@-xbbk12`d3 zh-?GfQ17cwHg@(HkPRyVZ$gQ9)wc`)!J5hrokG2}oe-0=!6&OVS zwbX1w%Mqx+H8S`WY9mH*i$AY;l$kd1171S&;tsD+h7<1fT@fT*+kQ8`P#`gjc32Bx zr5bBZniB~=hqf_r;?z4NLSev7vzs|Brni7G`J<%UQL3>V<%)hRpJwccdJsIL2$j&^ zya1{Kg$!+1bFKYE-ZaJPNxyI^TKK$$y(flJplkfs?XT0v9NDb0>gf%LwUb=C#q9Jc zpu*rmj(4&9B_L8t0QwQJe3%ja2>rL>`>Q{HXRwKucL4Y%x*&P54x*+D2&!HO5mu(Z zH)Gb|CKO(-l31>kYfyws%3`z=;pC6?_grHLP~+m&@w>vZsJZ+~a1fdn!5%*UH72cqTmffE>2A5kb!gqNsj+`&AJp4c17GYesFD`6{k62Z$9*;URUN*jn)S_0XZpiaToD}b8yr+ zNc|HXXUo|?G$z;=e@OsQW()kb7?1(K7D*S!`<=wrb@>d1r>Qb->XvAY@{{*= zJZ@az*_jc!%H>nd&R7ufF+IEs7>LiNxS(RHR~Cv9Uj~GV5@>Cp`@?O%Ah@Q$qM5A> z+FF>pj>NsbKiK}#)-w8K%7FNPxdLv`rutQ+SI3A$yB^9s%v{|Vw*nNcHz(ss(k%9D}!l&6&qY@Zl;*q>fq zbYB7n=nQ!ssMYG8bb)e>Z07~-`bUy#& z+-K?VZ&CpqM%=7T8sYyCx32-@61{!}?qShtPt<@j;Ys@v=pYib=_inrG<{K~6Lxrr z0&-vG0&}G5r{k|=HH(!~PzZX{iv6;T=?OZhTp5%b(NEr>%f0KtgzoiNJC)XK z?_!QhXEL=2V+sL%I#TopROTLNtO9fCZzO13NXQ0Z_;Ln>VXM0_mUW$AjUdYWI}F=j zftHYB0O)-|s)1%C{^PBw0i=IEcKG2Th-P~0XPaPWj~-_lF{2~q=zk(6!|3{&NkSoL zze2cF1C-|o1b!U=Dv=#$)@S@HHq!ndH&vzjtKmCy4deg-Fq8@iWH>_d6k?x?e|c+A z@wTzS63*PqOb(iM8kc|nT>g2>?sopht=h$^9<+xe0xeHyF0u$x&9ze?5xqA#?6aZr7NMOnvRE-KLM2hzs~fOt)KjHZ2%YXi(^MBF@S>q zE8&Rvx|zbL91%U`3=^MAKyf{sMEpEc0J|7bcyq3+bMiev)-PjdxDNhR^jVf1ogS=j z0c=}XZ`?<}VS2{z839!<^FXj?%Ox0F4*)Sse{wPSdkp~;_TVAU z1`ig%Von)HqZ3E!q%V(jn@_$3?t1sv-G~A7AVz6QOj0|4eB<)P51*bok->FxwS*}r zkPe(#y5o+~q0cVfVh?EW8B8a9G>lRS5^erbGg9u`d;dmkh(n%UvzG&AKZrL3PsDxU zqVGs$-yY~{2P8g&Z&Fp0^3B_ePmaD2a(rh<{BTb7tKs}}`-_~^D} zuFcVc(tWmi(d~!N`xj;AqvY%@B`4h(c}9OEuuIUO37c1;`Temcu9<8=29QK_S4tht zO#_nSIU2PV2{Q%2{E|!~k6AMtb|-!>omP$n)=R(Q#Mjd^F!k95YAs1`!m zIZYB~BZMqco*BS^3~zLcf*B@jlugat=8USW zS~&%PZ&PLRNd`LpuD5_Ob5*hgIcOQ3i!l{oj3xIwZ@f5)Qf!FNmWeXFq@KOnEPSr} z=NoO`6OqL8cCQ#A0$N4ZhFyYl39WbzFa3e1l~WU{Ou~dMOSg4h7OCypjHASo&f}L{ zXAcB*-=DWS94&NRl9cHUnuN@=>gR8ST`#`gR^Zb>CB%q1&6b1aLPdFZk$Dxy7fOYG zp|>n;BDW~q?>F|&XdGbCdRzhVNikX)CFJ&nYp)M1g%uD}y$PUCK!xrNz*ZgrKy%TR zQZpy-_{lQU1yM?#?-#{LY9&p|M1D`H-3&3#a?P)cqNUc?#q2${54)veH_&^MrB$5S z4vs#Oz`LV*VU#vU(dnJ~XN@u5Y?AkWeXZD$Y9C`Gh6J_w@zf=T{X=hs!06b{+t2s% zNf;G-ZRWMl_rc_lpv#r~Ptes#$2CbMLFZ5>pg#Pcu~}s+ONV4@B=|KgS!##bvg=J2 zYH5+1`-Rm|aHGEF>x!R@Kb!kftoHQ!RlRDnS2rHElb+y2v%{y|uiN>hwVoh`h}8Xa zQ6^MJ2RdG>(hzM0f^0V(xaSE!?mGZH?QUK~=!+0}S&D z`a24AtW|N)hZgJRd3T8J`b>Sa+SxzwdUV`e^^o; zUZZqpl0u;Im;fDwXn|J*%`!=H642R@u*JGnH3}=aq6|W2%GR;1$IBlIkPH&eEdZ3P zH{t!GAvplWF|~F2I*g8I6;F({g(%flXVR-it2G0qQRMk3P>J_U;!0=EF8T2uu7j+HIy%|jU5%o?`!8>MH~qoRp$&45HUSz%? zf|+Cvd=xv154H70o51$hwG`g&l#wEuO$ zp&fQkh!`50a-F#MS;$)A&XwpQcG}kSA6{GViOBU;c2?DTHwAwuLxEQ3m7>u7CD&*4 zzZ7J_sk#p+@w_9D0T{VOI2kK;+L!Babw}uW=;M)9OW?dG*C1s!imGZ_*;sfZ8(sob zZ(e=mh7resE1j6sQb8gBe?m8s)7AJfS{fAkW!x&sE}p1wjfK1#r_V)7CrBbhu zPD=jrW5=}2XNRtTUa3AUzBYX{oDrUsfJbLykUtVS@({6d`i#5QY0}HMBc!?>nmg%8 z#sByX#niRClGEn*dU?T?UujlP#~yMJ->077`0`uWw_dm48BJ zRF2!s%vo?;xvY8Dtzy{7IaJ%)3AM(*Z_8-k=*IRk{ln4uf^4co-F-5HVqNXtW zwxV3!6T52*#8ZpB5j;w0)=Bkp!}!3O6HDafZj}_g19*yj@^yMI*k3&hQX207#(9%JfRY`zCG8sN1s({`#;_lWr07* z`P9tly)7bZw7DCM;*ozQh%^-mP73z-p(AB}$pvRorPYqmdKSv5EvEOj^2jS2-UYc-eEmp0JF%;_-qZF#a!V3k4~GQg-T6>r*%D5KrgSg1F*tyZcmf`eL~ zV}iid#Mbj$@o%8z$wwY7;FW*;BB;In`8_MpX&b@rKC)0W4zUmG4F#> znAx_J-vRMw#mj;#bOt(G3(%ZNAf8dE8l=bf0vc2Ou7vp)l=}~4Q9eID(eDrGpQ>&j zkMa(?KUWGbesQPB*>Gcam*e{PsMqd~cU~jz_ZoN*+W5}br)pRapEFJHkee_3(w{%| zl-haap;Ay8_n z^G5+)FqdZ2D^X9yN2Q8!aI7x<=V>vr1^x_*WgwYb%lnkVX{d8`@khUV71yvJ8>Xp* z?f~}=P8Wc;u72w9`O)_NP0+Pt)`#n-=E?4KW z-@l!T9J^@`_iq16=F2a2iHhnz;@rM4Ian*5s(sR#6|tj0NHaxQq?$1rSsHjDNkP#5$eB168pU}EZTVOD-jNt-uN zFA`Ijp}8YhY@5lJi^=q(TT=A@N$#A-vss0KPUeEc&5-Vu#Kb50&4|1;;->kV zs)DT{(0it~)a)Irf;yVtU;9>7Aa(xZJ>T4Az4q(L)s)q)Z7R=;y^~2&nG_%v$rO-KdNq)$Xw2QDfIC}{k-nzb`Y)rNTsON|Ci2d zsV5ggUbOFGp`SEFsI)U&qUqH3O<#4trh$@b_O9~lHhx2XE=gW9w8ntPnCdm|Df}~!71tiMPwx<0`7fmy zOkEhwqH=%5&J{Lt%%#?IOIr*FIL?_5XLGXJesFKn-Dg6yAX`*;F?IT?&bjHQT?3Oq z?+eaJ?}Ua+rHUsNes0KLw34d91>ZQ7>f1s|PMEPr+z9~QZ1 z_!;s|0rpt7VxhB;8Wqpy)@@H%jUbSc6w((6rE&m?3VGUZO)@#AqWZa@NSw zNM!9?A^wa!1L{etj4pwpx9CdDw63ns-AF#Qf-jG$?&{q10j$y#XgxoAn|Zc~@0U)E zM0JDqrO92EVv6lx@z!m_#4%RA6{5HM-K3~G`*|jZVxPTS!gp<2P-RUrbTKrjq9af#LiADl4A%4k? zX!Ul1U3@=F%FrtQGSjQDoA)o&N50nMACGWSC|J-SlGzE4-Sps`&*LL+zJC}Rh06HlFgE?r z%5ykdoK?T~(k;z4W+A8!@TBY>h5oCPW=zS8B7R036pfi!Lt^+spbd*>C=Um-0#KZT zd&2iW;nb!4tK#D;{-zMdbhBIy1t0~b)5I{EG=NQM+_*_JJ;M$(2qgOeeZn_<^}Dgr z)%FH@A5Yglx+cFnbogB^Jb=$or%8ji^c@0 z|BI~mj;Hef|HsQ73CAoW>u{_@W>NMyAv9! zfeuZ%(tY%EDaz#cBm8JXQF00Uf>wR-r#_do@8gT(qQCs)&%QG!Du}lumYZ1W{%|Zp zpXxT{5nj>}8z+&F(pHWjF(y-fBh?-SLSF370&oTwBRzk2ZyXm~<-(8Xcutictjzuz zdp}jS%hR*>UFKj@P)5yx8-67XUtf)jyfFE!JAeOI-oJALTMSD0{Lhd&bfM`^a-khetDAjfMze@NGTo^BQC|i$tz|Q}g{3r6crApq`?yc!8MO zfOW0PW|QgwB9L&8#`(1=E>|%)lahz9zHSu7cg6unBil7{Gi-rqfZ;J zi_u*j)`52_JO0UXmT>1Yev;02ua%H5UH%E4KvwcRCewI{oU3bQfi~YVY}9ajIRV-O z7B!w_1`78$-p~}k&2Y%j^5P@Qzsd@ z%N1kJ_oOHfq!Al8z36n`8AWjCL-#uZG;Xq~s z1tSEI^dnh9#ST>~pWTy3gzCcx`8=R<9JmE6%_ve>PNzY}|dlMh( z`u39@Kf#*E;hweWQjs8Ns9re%06}4fmd40u_nMBLqeZ{-K;MIOCMEMqcUfyx<=E`x z@X`G9=atrBbmw#kF7F;eIkN!pV`ymba270mxJLml9eHrF;);HJ=K5}n1A=Y3yQf;< zb67bX3f=s8u^@`pKA(2eZ$E_d$)W3p%>8q(48Rt~bgrc+>hu09Yx#bq=Qh;HWX`Ha@QN5@g~!-+so0V9@!WzC0N-`+KIt_Ck}g0m#<=@oYC%Gm+HFf=u&lu}B=zdHD|t6Cf_;vCqD@(b!cDIJqP^dv4Tt7L~lO zLW9JB;jKQPpInC{y+jYq07ecua0C<3hZHJ??7f2f#y>3SFY0h^_#BeQ9|J5LdknqB z_;*I;&K73MjARy^XZE)up;WmwTIq`D)qjIvy2UwRnlyvNfJh?z?Z*g{TCYQAck&!r zeN*5A!N^#@u)HATk&z(^RkYrNavCxDN8e9L=)0LryefiLLTr3ArG_G@9Me!1MB_bF zY+j<+-9%kf1`5rXfp1-+asI6Vt{Gi1P|B$(fpEL%B6pXFjxAg=nHY?SoA} zNjjLf%fb%WR(7rca zq46j)0oIB>L_Sz@7Mir$Li!iMvS{H9JZ7r#_UzjLDF+aSn1s$GUkJZ~M1VPKoq-!7 zhW4y$VR7fjZnUI4a%VZ-ud0=9148Kda5=<55S&6O7p$1R3YuwGTK4EDm`a4ZTgJFKz?>Vg*;V?^-vw#Wc z!s|*=OM6W4ohTW`Dx6Og$%bh5>BBK&RRkw0*mDhsB>@NL*r*kuaAD}=TPuT=D>D^VrP z9@2jX{N|UW3n+*~ZjJe;(7fQzS%By^IuriE>5yk0nd(uhAJ3jWQcHBI$O2$+`g8&i zZH7QfjGYd}`z2!p1HODwuU%i3yP)JG&krQMR{$T1s5~4$qUb$k8^t!o1`#iKT^{Nx zGoGO3DbJvT?%qb#j)U)8jqiT=5&|zOgL!sPv{!`vxS8q#cqkipnC}@Y%Ybc@Wk0dh zrem6Q)zbU=)fDL{M0Wph)HSRmy}@y|IYIQTe{+ZOXj`yhm|wjeAg8iwlBzMk$xcreTwI!EVU zEOh-p4BD7YHPCnGL~mVqZy1Ebb8J@@PasSDhu@3BAVtVd zLUM<&0NQ6lq|>GQYvXB=_qSc@ z>h|YTrcb_y1^PelPaoy=yC=SVMruymPQokbcJ;y0A^Lv02!!Ib*)f`#l)k@Q22Pa^NP=Qs z@K9O#KY!X(^xV6!0oMO37-{cZ3q1!Xy!gLQxcF>9!N09fBihil@`q^n!@5&s4FQ2A zi*?|e$e2SKB*0&06~#834J_BGcSnGWpSgT2MS*N=C)DoDvs)@I^yiBu8ky9=Gx8pn zx&wzVw6@CzshaR>SB`pQmjkDdzl&oBc>Q>DNGV8(z6zMe>pB1ZbA)x>j8Izu!Ok|w zFZy6jYo_7 zx-oZrx`6cmJWBnld9E|9(~Br`3Xa^!yKZfFcA;@(h%Q#S^!L#H-dJ^+!E&7@)8ZeH{kK(^b+PTM0{#b&5?YlW^j3Z3^M+B8;gRDdDJ7~2rkVeL%t>b z*9jd(tmyh8`udn>IQ)cE`;1HwJPRbkn!s*1yFL+YCyAnVjtgNEGalW_spDflRiHn< zyy0Dk^9sOlA9!K05tV$bxyO{aW&~xmFl&8gpo*8T@hlYM$3`m^WLY&s22I!tQxkr4 znk#QQB8CzeYz%UvK$g-OS&+eLN~Ea zU^k+B7?~#TZ4gU3deNlL$AkV>B_rm90q2(1Okc0(B3GX8o zTxq7Tey6f~gQ%e@u8qubwMyxlr#I?A`~vHtka9jzbHsn^1>n_EH3UC6gyIYyZ_JO6 zs&Vn_D?4#fg!?~u)lq-_lqWdq3XPuLXcHD&g6dI?Jw?5iO!v4Y2(+hXQ!Lkg;B6yJ z1u%rj3d%hB^Ej^fV=oUpXIMzx+Zi)LG$mQb-pD_4yALn%K@##h|EqYK`LLU|;mx5) zb}DP$`1~1%3;D#C>o!YUahJ@`l!E5X@VgOC142OVetxMGK=CuQX63mW4HVx{|=RW5KK^RF7_Xukrl`;7}nnNVB^F__zLBL1`ZP(NmV z?D!BT1A1+#y@MVqFqDPUhyw+j2(k&ybVJVF;+Kc}GQtdk*Gsb!?&&XiP1#!H>T=L4 zMo|YU=}qdMf9_(C{qzK-fYpu)>a*X`Z^0J1ab!uAnVvs@jV_l%s%HGt(q~;7eyr!` znQA{3|DYxXtsRw9hf>l{Hnd9 zzj^6)!)b1Y>{|=*b_{ID{lBK^<|g4dUVGg%VU!OGajg8a27wo81ttzfkP;DWyyHW0 z@w_$}vwYQ~Q5Te?H{8>tY-q_yuqO5o>tRI@?dl^9L%kfNqH!b(@sK3|UV-V!ImKj9}FE*0L z_YxFq6p@I=)T*I9?P3MDo4@&Gp%!YfCD10u>p~U;(h`7LkYV*B)R63m;E096X46FQ zVy_3M?QJm@4iwTu9Z28m@b^+lP1{V-$WR6&P+ec74}1 z$1^aOL($R?Ih5vuPse)Z(Dg3$$vV^a70ygM*-JEzd0*c~Y~_67c`4BByMdgbIRR*GmE1 zqs7tbz;EmY1%^$8qr&4%=pxU)h=Of4Sb-Nx2ala9@e%KzhkEr8(2#NJ&8HX~{(8}K z;mZ(XEXjl;!a!nY7Ck+kH0TvLDk7pkV$!04$*2#GeSY6+%VMNRQJf^3GyV-tL^xEitRtN|~P z2c%Mr~l& zn7392$?ga^-n{_tocij$Kk>DWS=8ER;o>S!0lG( zpMKE&#z}@UiGDZ5%t@w=&`Fc4^f4S!VAS5_BN7yd>HlRNmS-|G{eiqiQh&2A65tdh zTy)~eUj<*p4cM!&`?K-B8mlzA+ng)&WT*;H>`>1FluT8yE?wR49Qt; zq>Kfd(4yt_z+#sNXIw2xE+EFiJ8Sj@ zmpaoOMG}$kl5U*hKx`IQ`qW1(Rq(gii`-*nI@p}kA;k|IR!G@{o$Np9#9S z7$r0Q5|p6Nkxd|QNC{FT)_~n>IL)F3REr6^F5U!t{QWmm$z3zLQ76=M%~ct(%CWtH zlWU2(?;bTtg5!_4IjAk;XA|J-_v5SW_t$z%;et2Y`vJe|(H^ z7WoF*R7c1*yb_l|q@m)H~<(sXXdQTUPjo z?D=+6SVo;7%HxC9?FuGD;#9FWwbqvY9|EN(Ty8tUo0{rbDhnkD1%j^f=O*N7KjoI4QJgzLy#bkI%h zBop}*haap6E%W)0RK_alZW-raKOsli?cCM(b(+ZC9J_vxip#VLh|%}nTX$T25b^Jw z{H4SlD)A&acm6To-LdITl)`Z>Xmd}GdCf%msOdaeWs<)oN+d|)`gk!fw{iYBk7!3f ztn;LKfH2}qPCc?^kS?Go`gbb0psz>sq!nyXH3$#*c^4t?&94pZVCG1$(9818iDGm+ zyc-&4!LpH+oy^R}zZz;NXvs~M#9)qvg{4%J?-cG_09ToPf2gI`RLHs8FR2*Brsy4X z-E_E~%86iIddYAYtzEOvqfQ1IH~#0=)G0+)Wd2i8$vWXIB}@tpB=18wf?F!aYO|dU zBI+m;rqyoy``g~$* zezk(DdMvB>)<#gJM}^tX=Tm?le7M0QeR7v=Rd%*!Y&xdy<|Fg6Co(%X=q{QFejgQ@ z6Xv^}l+U~;FfNsjf|bO93NJA|em-HE<@QUrp10pOi8+Dl70V z2PucIjH@zrS7c{GioW?x`aOabk;C=vbB?DBb8|{0A4a+<9~^z*A9Z$l^>A^Z;|%e< zz7Nsy%(vLLKx%dv3(_4;5*Be6bO|n-3s79Yf(n8{<|)=Sh1LJi%CLSalZ9+*P8ngf|1RM+Ac`<+mdG8>-_M-kLH=F;!iniuvr z+Rh-mK3iD?ANHCFHm9w%=ZnMl_!loOcqWxv#EmdmFC`!L$}%r2=v-d8$RM5LqgL9t z6Vl&`TSySAniu$f(Zj^lEadc1F39js|JwCE|06}{1tP=~|MKD6rh03?OP}V|pJHFg z_4k_P7C&s)4@k_VuV{bkGG`*|YL(IWjO%_9d^9FJ_ectp@H78`&7Tb}ZP+IVgX0l( zCOAd)1ofQgiBqQwENaTRwCL*Mb-0WW&8w}Ig-V8+>9EmLGq2|7H-l_47Mz5V~m;Da5Ui>nuf;$q)H z4oh|aos2B%xI7}9A|cW={Qb*Qg1xF80}}3qARcdfNn_JQ@q5*@ifpPv2!xQKCpq#* zWTfxY@y)gc%#msdq|wQQH-HCC?VLmwguxBGxp}$CaH&6+mV51lZasG^uL0M)sa77w z2gEU^Grr)mSb#c&-+XHmtkSSFQMYj;PCW3nZ~O*+?hZ3T8pfH zm(Nf^#}d^R_Ae*mnf@e}a55oZSb+m1YTlG0GT}-{usZ}mAj0Vz6;5d-LZ_Ha6BVNo zks6mydrg+X=tth~dm1sEq9S&hPyL4_v%<~^m3(cucdgbM&R+cM|=3qt1Q_&kp`)sEBj z)=%+yw)3V|blGR0Rv0FpIHh*RlCmd}LHYvc$$8H^I6`M~7-ykTTCtcxWAe#x=z6v8 zPN~cLY-74pWUh|d-7%R}uq!F?*L%(Q#*{E=8QgMWbto$2}IgvL4`0tG8E3#GeoM zEvag8&g!i=T@9Aex)V=iosK&)WQ%{3Q<>2SJr*~^EOH4nCmMpY}{&y_uw*JI9`|gUDAEO_}Q9I&`1%~8$@Q0^>6Q{RN01-)Kz-z!tJr$O_BR4f9-o` zb|u<<{uMmo`YSv=HNzRl?|jeTvwjfEmJ0&MVxdNw+n>Qtt%L_#{1tHW;THCVI=JU&djtaCr^&b6VP*pW6=;nIKRgL4yi{!|NSWs!2!!SB&2}`T<1+a4%yW`M-kFz{`_zn&{I0mr*iU# zQt8t$09gC&K(-Yk^#hoKoavT9y9UV%f-<<{XMH&2y<8UctT>co`3rSqzl54!(IoZn zrYXi!v{K`7!STA`DPFHrX-6GP0H6C!J_yMP%@;t3f_(p>nE(8VT!(RH5PlRktjbsq zd*>NRI-n%%E1-|Lanh#GMBxdbuf(!`hq4Nr?Yxp6i$<=SE8iX7FbPa-7jJoOb_gUy zPdMK8cevn!Ujqkf^%lZxg1&7+u(Csa2v)LjXN6RUUKN?0;E30V0Xn)Xb`h zTCYV>5GEkE;qgb}(q&UhusVD`G}i_ivIsd4=-!gHL>^Mn$EFhNTiTEb#gih2Mo}0-n1z z5{SA&4u8-PH)@H@OyDo=|-HzSXJLd>mYzjF7z&5n> zmCHFKh-Bm(#tR++XSHUCtT0e=t8aln_to9%y`wP5U%f%MwwWRZM@OzlzD3Z8cpzZ= z!87Z0vj%QvfyyV0rdS!}T@*Ax0UCVZ2r_~E$qkE(5S}3SimPhkHr&$3yl_j^3CLgJ z*=F94i=6X}mjZo5%~0|i^80|n4KPcK@#z|+o%dTYig@e<2)`!%0Z~xDCWgm^^1cagw0bRjW#klXE?)Tfi?ia_A zJf>NZNmHpI2-kO6m}h%>1k#53p)VFO(EeEVkEQ3B!N*-< zrF1zKDAZtIlPpboG2NyFakU%7aQKpTEtz`*7{D9NpSyZ{&{Ps}K3qi55tOp2WXNYkO9#&5(Z7}ZlF57vM>MpA)B-5ws#D-{gp85{piv22 zc%F{zkZ&1s=`0|v5hvdbZDK;A2{`-?gcL$*MP<%ulE)7Ec#U@WU$ubxG!6<4F+uF& zXIokXKLW6K{jQQl!pqvoY(Pe`!NXiY-2OvUT>o~)>=ydI{T>XyYiLbi z56BCdbSk)6|6!@RtdJs1NCNqc&wZN@-a(XfQF<^d=)fPWj>;($ott$PSB`#TgFw+BN4k7|&mwIr6J%twi&ResCL6?W9=}h>~M8Y;LI9-?D z2#Zw!N7VHlr%?V%SApdNy-Cx{5L`Bf~;D)mR>8lRqi=LS=^@D)My9d8kildqw zILPO0KQU*w3j<6tlrB3nL)rPx{&DLfRzlw(3|(PEgD@0-mo3N#7+d;8-N-uRw4(8R z;R}@fboz`W30dLc_yj&OJ&8Md^k2xssUrMuO=>WTfKrR9UevUN%X0nOA2~j3xs1Ld>DZIO(aUa9rA72ck2ZbqUeK<+v0U|5oJvrEH%?+QU zMfj=TM^S03U6Xz0ZBm7D84QS_;{L%9eCg!hTR+P)RKRf?0TG?J92c#ZOHX zd*}ka=9BFV5(LA`$28Y8yl5e~T0!#^KK2 zA{sB==m<-M$y5bAcddM?(N4n2D$cSP@4hUp2)?rW-wYtW`gv#K=+6AlR~$y(;Qb81 zWfq(xD;e>DzH@Pck%tbh1Vc91qF+0X0P$8{X!f2|;L-L^ZopToeO3Y8O%M?Nf{gr9 zJN=YJ>?|@$?V|(9ar=#5m;e#(`+JC{H=mGsH%E(GUjmnN^c^dm@!)XJwIlg_t^~=+ zk4{6yX`r95q+%BNY0aL#bEkge9ySx17wH=glQXZm*f6##f<#b9s@F4IQ4`Jv9UQ3_ za60gYIv1Iy>B-#5Sy*k(7;ryf;TqU|@^|+M(L-U9o6LLp?}K1^q#noIv@fRkat4Vn zy-!otP3Q~fEF=W^u>4=%-!$uhISgmI`yT~|Y`~OC%6oJuG$<|aN;b;pwH>QfXC*#S zrWS-*iy}MTrAm3z9bO?r@{iM8riPm!91J$LVk=AW*cC$yHN56E! zE6ONSk*|HEtC>AplYZXU1d*01QpFOYGGcJCV7Fa3F< zKk~B2pxWtc^+0!sln|_gFH}1)spSrzR#ZFKTxK!chk;K5y^~!4>{tmc5D!sIJx8YO zogAqUOGAHL40)7}6O*+=X5B!s=B}<|m_sY4 z4}zj9b6|rXyMEzShHR}BMi)oo!H0RySV4!LMROl2XApcIz&+dV(uZPS&s4Qz_7UUL z^3hGFRwA}u>7zLe_ccnko0^cZcxi+-82Rvqh`wV`B*noVB`iiIe+7EAlh+@hf*T3` zNicg=e?|66)~&9;$e=%DW4@;-FmUuSBXs?A`Mc?3R*%uVl$?P+SO80{HERohb-~+= z!$0Y&`F9R(T}hr-E*JJ_#hI_AWj1$?#=UHRq>b#--t$wY#(ac>QS77Na{+e3pdm1E zlJXO!pzCCMiLt~e(Fg^fc)Qj8zS36Wt16{sHdy$QLp$ECz;m*HKGLl9z1i=$@MmGF zg7mMqlG7<<6ERk8Y@VG{S^I3xwc=AIb-=`R8bY3M#32Bj_m(b)b1zQoDr!?U zG<1!Gh}{XO0Qnbr^d-ysrPL2(J4rO2N3zN2R(r18V<|)BhnewPqtK{q0;Q17>75s^ z8h?%MB!12l$e|U`j}WZAFMooB{R%>N z)ZpzhoJSsW1IUVj%WvuaJrHA_BaZMUyKi8g(ZAFXT-UtCf!_Uq=EJU*6&d<4GwC<> zh!QAFKRmAL?2z-`%(CfCx0stX_uUy>yq|8$VP4}hrR*UqR;#rxgf{9>3?-kD%>&jJ-v|!-IvK z0ux*ZLVjxAcjf4DCJK-cl$9W{J7!gJYLkFMYmTaW9}6Z^9F-SWS_tmF{MUTWZA!*M zlBPQF=NOCuRv+XCcXjT{Xaa`69e7B}Zwb2V9uE`m#F!xvxu~33VS{>N1H9|eF9VH3v6PSKnr(dQ;h-|d4 zY1JfX67I`jW?s_a4A>Rtc>|GG3g|0SW)V}a*Xa>9i+UNo(kiS}9--P);SDfIA{&*> zfL_mdMe-3&!!Yj{Z9fce$Z>lH`h)mI8LtV&q}YC215JW;Q_B(JKw+k(|m)_gfqdZi+Igj=w*r)VHRv}^$3`Fw#v+>xNgSGmbgYkV;rsHh| z{urLNC2)Bf?!KhU)^hE#iFC0A1-ZS|r*p*( z8Nb%!W^z*b=?ItF0D#uado{?l%h14aqB9MHGTC_b-QMq;zpAy3LE&SCjb3vJ=%(sX zUy7blyrc`cnTCV<%Hfph3Fqy4G(pU06pccq>O@$CA8(9+T}FQ=^(6zo!nKeJi}9ig zrw>QT6C=uFEsT+wX9Mr|=s1jpy33agwY-he`!mL`(=<)hNC~#^ z|F?(+k0|od`=vmTh{#Jf)fCw_I*)c`O1JSt(Q~g>-;JzQfA^F1;m*35XOK;}vhv_7 z8`bLVcrDq$vC_Tvtiz|rF1?1$;tUE8z=a-1yc##rTXim;43ca;Rr;G17`}xVU{H|= z+{G>#6NUh@@Bl&H8d6$Rb{8+i?=d0Lq1Yy3k7~U)T{MCDe_;ov`OcKeTV>OMHyRqo zVU)z#uLoO41ZrhmWhz{s|3u z2f*p~bqJPEjyx7Xr2E+q(XRA?a!d8&{Uy6ai#H7f?d?-9%E@9oPib)7%dj!$rKaHz zWZho=cK$w%5+gOU$m^cRDsD1Z;bw#Ngb1Y7mdNdu;&QL$hs?`^mQsB6Qd~4 z{#zjoep@GNrYvi`35}HX;>*l?Oq5K1*2CTH+o*nZ-PO%6I_Uq=0?3_hD`!t^+-ag# zX){X3uvKUiN?JIq%5D*f^+Pvqblu++%M*_G%Ng|J-r%528y5eWHtSecq&SrMdmmC= zjq8M-{8&r&(JdTNvP5}a!DB*B)>jEG=d>A@tf$nMq;TL+52%>o(?$10P$Gw|C1!a;^#|j0`3=dZb-RYUIZb&yih|@O&3QLduXzJOrDDlHl=WGU^|%<2AKs z_%gB%!itI^Lh`AWa>XAi6;{S$CKN=|`q`cbZ{BH;xO7c}Q10`ALS|if9y^R|3O3WJ+_Zf`&Q4Bih452_n}G(&Q@a z`}t?2o;R8_ehdN6Q9*FTkB+kejm}QgHTDc0TE@!O%E*C5)k}&`i#gZlM@r|4#Be^;${4ni+<}S6DB;B3gP>c;q zy|x5Ni1SSO6-YMCVbC-fRFir8j(i*)A4uXeP|zs%7&dU}Zx2)*l~{s973o}OX(EC$ z5*dY@F7FH&F7BOr5?Gss2tYz6V5k?)Ac0=W8%60>YL>g=;%UL3qi^3dN zVf@kjb>5WSDnmu`Eti$aP5D@k%63;eiRg!du+a$_MCj3@dgxCd!X7<1Q5T}x!Qu!X z2cP$#_b6n7C}gZKy4%ZgP3(|aXLx1cmqn@5uxZtUjf|@@ox2Mna-BC%l{{+udOw!t zjn4MIg3j*;2KKjPIt&IZr+ogM_OzT&x5Cm0=exPd=XHK;pUm?7jqGn~P<=gnXEh+y z48BMS&BNw!`uu$9djeKpsITe$Aewp!ynZ_}^Vh^PGGyySE!s*ik-1*T?6PK(GAbMNNS#ph6)K~L z-`X0xoKYr`^*pZiiN6J>O8H}a>Xwz45nYC);P>v!9*agEiq07%Ez^_IZy<9!|M+a$ z3o^H0H%iP47eM?vlEkfri~*S$udUj=m63n3MdomRz9gl*!fvk{XC*U6+q<5ED`BhE zatN&zvh!Tq@|JQle#2*b#(8aPx4umWpZO%`QVfb^uFzb!bH{7A{0&I?qpV~`o(qb! zFh58^hkf0TzP^^Sf}A*dHV#qTahSQmD1;Ew@E>Z`{JJpkoAgI>GNS-08^ZTq%X7pM z2WM+j)yg2bv=?b!c+a1T*DZokDGb^K4XOX@G_g(XeGVbjVfKX>!tr&`)K7}ln(acmgb6sk&=|yuz26`*_cR^u-eEN-nen1ajuX4-oG%gjQcsu#Q zl#W;cg5^|@yUMMN?$T`lYZR`sjJa19zmi1#KmK; zdj+oMquzD1*Ci=qRJuMlKEvrc4tKvBQ{=9EpDcl$vwMWidF(yA9H7#9@L*$E?u8O* zFN3_ECkqF1eyfR8MK?>#vlSnXG&C{dF|YJ^8hKw~iAWZ+_{owbY*E5yODZa71-Uu+ zv@WXMPRt2yqGaq-o3T+ z&AEflv;QUUA=$>Ik?s+UCqMQx(;9jfFun8A>471WrHL3JCUW_+OQA9Gh9jQ8<|>TY2IFarhGt|EF;*Pd(V4(Yy=^nh4ae`I5Y2gKP1 zWoa+ZXo~(hlZ_%mr^aJQ%h?;9hrJ_g_HF1abej(D%fR~^Ji(RI>v^%xx}uQdcj$zL z%k>B*=EGIdWa{~NZ0~mL`;&6>6b`iEeUSDUZ`EG&A!ce`h!m~8vZNVsT5USi7)B&# znek1iSEq4YtesTT`eox3o7Hyai8&o-&`SziKyRbiPXFk9#n=R?!DtA&XdQ{kQYvCt zZP^f|;NX^L#Z0BD3ZCnvsGoXaLl38uAVf+_|7|QYI7UZvV|xDS^2+n`H$8hT=)T5k z#!D9&ogNtT%s!?p7~|p#c@oMNtrHynijtP--qpDRGFk}fG1SbEbeq*&n5eIx?MXB9 zCM{%Nj#v_Jn2P;s8cdCv((fBAYvPD;Vd1@xTbvy}O?%us^_-P*rH*O!$+Ci2#h70k z>@^gv$C#;?AFJLzD{qOU#d1fd`|ym0qKJ&0?(;adJ-+qq?67jo*wQtvsm-r$qz8jN zE13rcPuSz${oM;KeDt+aSt9$2b-~g#4~|KegYPpMS%(UA`kS?1wtRZ)$*U`#SlfB6 z^nnLo;mMdFsqXK2Zg1ji?^{+3etu=UFP>P033wj`n2lK#I%g<-xa4}Y^RP&6`EqQ0 zN=3(IyI!sCp|Cyqt?B)((nDMqn}f8YTdP-upo{ojdCp+uEoAmw?rS_*1o>9^sq{vs zC(!^~CbBtIDxm|c{hPW1vgG6O6d8;w`XcLv+%j!g1tH<2sk;QQ@PGTQc%AQkhT9(C zc(za~cQ(zwz(^&OTYoyUbDQ+4jCsll)eiTel={;uC7EV8)Nhm7Jvr~&9lFQusrWGJ z*FWs?YOX$LD$&(E)3f|dY$T_LFta3+?c?`7@>JG5o7f&I``r&4?_E1B!t|{oQ&bLQ ztsE_lRJ@7gRJh@>gIGcaIw(EAj(~36xN#NG&p`!54!FHI*d=SE%Y((#;y$xQyWs}8 z8;-H2vn@*m?^2N}gI3y_{UHO*v==IMVg3!Fp`H_sLNk~O5bJL?cidy`S>QyE?WV|Z zB&w&JJm8R6kg!V>Qu$30VPL?cGTDBnz;?#$&*jmQ{?BhT7Dm0Vk4JH4x6{QepXKwt z_NcOWaw3OMBqMn9*z1f)UV-;q^emjbvoBtrJn1P8#>AUKFDTK+-{0Crp{lL9p&|m( z&vn=$z>flkR!qG<$!HoGCJ)8HW+Sy&^YZ$bo-eDsSIHrGjXU2BLLFECtmz$@GL406 zU2Ya#_Fu__^2`N&s*s1TC$tD>d1vpysA_UNN$KoWMS{@fi7OKGJxup1*LROm+wOVw)jQ@^ygq#+Jte9_YJ!9#OXRZ6;-s>fMFOXS7kL8Y=v4(# z3Nya6ZW($^Y=IWi7&n%yAS2NqtL}cqlDV(S7KWESgb;wVQJ@+)!6b( zfi^>rYnPs?b{whZeeBrk+Zy6fQgQxPEW*unaH;jP8Kd)l%NC2xQBuX+Y~sMhN-aXh zvBA$pRb4-x39UF>nW{#gDBE*BZ^eN2rIsf}dQlP662+29?<-KB4=$`zh))d%m{P%j zy9bx|XBq4V^AoC%)u>H2H0ZwbqL7amZjEkoMnHFHz;*II)Ioe_LUUA_V}LXTfV^dZ z3f16HoA_$cZ-OB)VR4^)cO4CWma}!AEa7vY`7wVa_1jUdZfQcrQj36+p>VTE*2wp) zyb&RZIdvWlczO{fv(o*smPYc%V8@BqRl)bV-fnVvXDVheJ~sElXS=A!b~L87TQ5O4 z;Y$%^DCUr9qxp>&H*-qFk5T^lOk1;EtHyvHn0PMUwEHu}<#(e)xMC@n|Ll&UM&;h8 zWc&VnWdcT13&$R?)fU~=>Kc#!`{cA{aXu5Dujc_LF@v|5wR>*f=CNyp4ygLSCT7rF zX;IgYF@c$k;zJ^l7c-trwi65*i2MF<(y*XKkuZ5mAeJ0=qW-JB~aOl+%gp$5p12m-v0KT z2s|^_e#U`s$T$K0uv-W({>FLgtEs|B(Tcho#;MrAJL_KC1%uh=!(qcjQ@SyDGmWgE zgAkVy(0~>+I`d&qO53m-UzN^a6qz;2KdQs;KEELN^UR)o-u<-Yl%;299hmd}6)8dc zQh!1hCuaCNzFZbN^NP5`;C<|DzKQ9`PB(a8rZ0lVYq*Aw2xJR5@vjqH+~eJzb>-sUTUR~p7FyoMTvv-6YGXf+ z%1%;Qt|sSW&~H2`^Q7(l%SF$GACIWz)y4xwR|?WOcv+%fZ0{HC`F3Z0)Se)$!|nSGaj_NbK}aOjhQYm5o1D5;fNyj+atw3yKa#McbkV#>}YmedhsizKNq^H zw7i98uW5Y`S^72+G&^?mQZ)}?j?l^K(N93DIKEdfnU|)@bp+tIG)nL_1wC?Fw@D^- zuql^{tSJplInir|6TIq6Ph_57;t5g}yzz8+%MXQq^gZMT{c#>el18=3*;Q+|`%Cm1 zvs@vIhI%omlM+&bvUG$0;hruza*ujW`0i?f?a~GY%>L4kd_K$^VDoL~jv29@IM3v5 zlBCSp+cOu+9Qd%hW#-e{x;YoHIF;n9{6rl%Mf31JaO;SI*#i$#k66``0Vi|$o+FMz zk0dpUN5o>}T8*Ohwwqgv@Y2>~S}Ee0SzQGEql$Xv^yr@_5+Y9rRMKUGeX0x0^TDqw zUaurD(r}G_t@dzW@;iVHC_bZi+2#RqOTx~q=|&xEW6%f|AkwGLhJb{w@w!KIP=)yE zL*ZVCY*&3nW-?k(c70spk)l+<#_`y#9cU7k9nW06*A~(pZQ@$2%YL<|%tAymCLFUe zQGXMe=hl#HE^zgCI>+qQFTj*`!O2AZ3i!vE*+;5&Nawgj@|X{^TMKDmIyYu1&bR1j z0e05>8q)qsoyMw55@4Hn`FOp7ybtCAl&n$yfPT(J&uv@^T!@FizG$40dqX@RyGr3Q zWH!qEaEDlnVD4zyl?H+eh;ra2CObhF%gBD?2B6``>F>7%;mqkQX1xE@uEzP+RjyDD+J1_Y5Tz+AJvmOBax( z=C3Mfud`M$lZfkMx@P`r6R-Q?k|h&VS`Hmo~#oR(d`(0QUAH&$hwMz z#ye)RFmV&9`cu#a%&L5p&?k~ZrjFeQhZfTfE>UoE#sRL$GdMvDW1Ne_Dmu20!)%jP z_7WgopZq-%I#kb?)n$8k4LQK^TrNPS;SL+R+!=w5-*879ctHRKY4$;A>HFIfB-h3o z_AujE-4%H2|LW)al zq1X&%)#*{_wxoy%)bTcM1jG*mn_3c z!MEfBd7BY;HDnyHtuf&6u9}WtxElW-^4>Bms`h&uCI%QlIwgl5NVY>}1D21$N+4H}zL4=T#t@%i3?%1^Smr`Z|4x(Qc*yo3 z(Af6ddR20;`oS2g2b>9qi0dfmcr1a9u@D398+2egV8AWtyD=AimRFx}f!tk-^HeR( zWeR>E^of{ZJB-p$fmISC@_GC2861w%s;=So_X6N&)_XKU#3pRkOiyDxCmK87A*V-? zB!?ihbUa#!V$eH>(j%mN*a}jHwS{aP!0roSnWjYb#^B|@*bsXJP3DI^fj&zCCfP87 zOEE6j$5V8Pv% zfN=0&j2=gF`tNljylkPx{LqM2;4v!X6hKxO4teYMByG=tZ4MT|@)U5Z7R#Feus;c^ zo(#jOSS*jF{;$_vNO^dId_tx)y}h>mAecJ{0%6Cu9x6}SGhWyqpF%7QaGqeswkyQa zYSmsb!k|21HZ(KgpHev#c8nw?h82%ZBi#)zLh-}Z%Mbr^hrzCC%z+%Zq5Z_3GIon4 zO7(QV#vyhJ#gzyoZ`4AwMI2##JL2bc_<@c=7DS>X& z%*J|zhZjT_^-&Ka5pr{G3AWlEDBqrkf*ZO|v!Hk|fNr~Uz#H6yNvbgS+hQTj4}m z8`6LmqapQw`6SvL#3S>~e}0uU5uV6Zlc#>l(s#i zfk_hfQ>S3Hbr^N#vhf_2Ofie=VrjZKCjslqf59Nu(W+6vmmFleg`fWLfZS!Y+k@lA z$INkw8Mi``rs5K(-GlAoMHpkJJ>GNj}b+uydbx&{S0hJEOXg=wDjJ6 zXFRMY8)QATDi6K9W^Hu^dKM%XV=*!MetpoMZ0=tYyAMvjIVhR92RekVE9!a(C7sy` z(A#x+z=*oJki>#0+8j1OWf+{~|4B{r25u!+d5kN~-RuGoG8G$;}-DsLhM6G&$nE(HSMP@`;=I@jm)UUV3__Tj$8h?km1PM%w*J?}ezua^`l=6%Bb zKM!55C!q2<4E6}P9)?RXnJGY-gJr`!(D|l+?4X4rAIo`g4@UOr^RfQ_YsqUGM`_6N z1AQBJhHmUKFvURk%Yw*RC*ak=hM^TJCTVG;LK|A<7NE_+gy14{^*r@X+xra+T!+h>wk8cG2Jr2OZ}tsnaoCe_vIJTmX@R;_3btqyD_t;$4Ki;) z_HO|QU(FQ_^icADHD@Na-G+X1$-0NJRp62q@_2k}14Zv|tdIHbEi5V4`OyA(n6`A$ zK$r=&^!*RQ(|b)?I|K?LxsC4qFvg@jn3H`p@?-Ri=6%T9iFm^x55aJok!1=5HvCD< zXlQ#L{07czNrdMW`)?`~*(>JVW5ScL9VEl;HARO@xpDLK3`;IKvgyN@tI#+j!iftC z1ulSJhHQd-p1Wix%X846uDPn8SxpBe$K;iG`R^ zs3dBAUjxl`_SLz;i#-yb4A{~D;%3D1V>F`~`>z}qRS^7`_x8zQ7zG(In9~-wc`Aq} zX`!^>8i#7lVKj9}B6d*#Z6i~T;YQB5d zG&jtPv+WhS`2p?6dJ41cb|&H#zgeFsR{W-JLQgNTz}#R*teWvXw>xEX*K9;S>#hYw z!X%$FeXXd!{5q1ZBnhbn$cM_njj@{p$ymjn;GuhHL1L22JKRaeOlm0r$rmh2GS-(Q z`oXC86HwUl*x!)<&(uaKl0ZSxJ&3wAevu9iLDmB)`;&eOm>iBr@rG%0Q&P=+{SO@a z_ME$hm%}}pG-Il3$8T6PwyL7p`3w}cK0Kb1sD-_{>xkqRY{!vv&NKrLj_a<#H2Xo9 zP^n3B2Ma%L)HzYLpv@Edi`((oln;Ait6t74jE5lCpM7v$_` z%~28z`A3gm(t8!K7trrcU8PKWudc!r2?vDNlCd4AGtTjT5o4fak%YK}-%K{*7$-km zkppE)GBx)F2pWT_#64;e6a9U+vrl}FtllOjabn6}8E@zVO&cFXeD}b&Du2KNmuKWC{Dm5h=5_537ix^_MpYUlDWXsNd z(QIO%e(Q0feW7pIXWfaEr}oCpA3ZtLt!V$e8eH%bb{p-TtF{mpH1mU|&H3PC1!l^u zEL!l~4^2c)IDsbf&%tJ*!Iz0D*Hon#5?%9jW=5gZD;d|I^4L4Xi&G^IBr11{Bc9Xe zr$FRVNkaIFAX#zjDOer(6{B(Y6fECd^{?5xs$-%`1^NQ_$W{2x8fN@Tmk%c%)t4aQ zi#7+*&pZ^)5WuIU8zQV#>ro9HCEL#Os5RJ-NkR)&dcn}lzjt-L1{^!b-puS(`CRd1M0 zy`7n_+-q`GyE|ape1G!0$?BK@aHfkj0(+%R=M1WwL}VV#S-8h*0LAoBAnYIDG74@a z9Oz3QHjU`!zB9&$qytbS!%h~8yu(u*KOu&gL;NKgB*UEVER7^D`Z4C0Pg8#fCKdib z9%C#e`^`*?E$zJq!ehhPqE`lE^dG{n7_- zyy3@?H{Y2zUSkgXUbQgQf+s!IppWdIwI^6le~riWJQGC(b8iv~cdH9j4DQq|J-l!+U-DrHl6jJwrdQFVjE%qJT_Sm~qrM;=v_kRbE}x zGI2US#F}dPAtXCP^Y?>9%qmf7lEg){og}7&-#9sJ^{~YSe*os(LnBd9g1KqWzTP zmrpDDZ57V{3>}Ba=~Kl zN%+5j09AuEu^bMSaOAXw*i<8z9<5J+`qVra52SL}g9NI>JK9Aevm|B=I$5WHXsGyVAL7e6H}F_pkRJWw zsHp3 z3)cbFFkZeg3SOg*X`>M74qe2^B89qJU*8D`XG2JW`Vb}`)0hXlvLSStnuFVz)T&vw z{62G_#~~ zCD*lw^HA@FkR<@8VQmAEOvKg#;Cn|LjR-lyD4AS9XI^>h+n{A0s7?aR0V6|18uQ@I zMSs3R`3yP*+d=iIz1lFBU~}+j_ZFj&$hmkWCgRHoZsdFCJ7h&SU^Bms7W*Cj6ds;w zc~_NsS}yDw1iQ;GKhEqps`XSrzN7fzmgPhA2oJt6+MM4XCZ*n%^Nt&1{)p4}5BK#6 zWB-?^XvObE411XfoMCp5CLQV=X^6u{55YT6(C1$r%j%;z1Yse&SvwVsi*|1FhTDDW z7qpD}H;{v5>-o3k(d&@peU8XD{BQ~lgkLT%YDxEFy==jc&T^NA%uw`i=%;~VM6!$A zq=lQL5KFYTY9g5vhQwPl7}s;`em0LNT>rYXjZSdG#54ZR@6kjIXg0W)IMn zmNBU^z!sND5|A`Y4|3?P2XQb%yC7zsW491W%Lo(A3b7L55<-p)3` zy;0im__MG7ef(?b5>uty$A zZ-f?x26VAc5T>O1c%meP0G0V?3z-?b$RIf_m44Eoq?tQ+%)j%A{MZjxI>O!&{-lTS zRiUTj?&H@0p3IEw{X0HX@PqPXYmO_$dW|BE#BHA7c5yvoaOg0!)uWxwaV=a+I9Ll}}gsb7|2079n1#5nv1a-#p!*fwWqiXu2A#03m(uP8M)~%F9y83;(eFg7PFki3c(b zB=6rcgS^zv<&DDa@P>&Q)Cpc~R0=%|n&650@dl$@mpjyE>WBN<29hkPRC^nloJ53( zpr=ZBmkDU%;coL24(Jg}Qxx4?dX&Sa9EKI(4qC zqUM8**fBW&;M`!X1EXcGlZ^*h#{63Cm0gO;gp^t1Zd!WcQCKN1w(rxeC7`Me$pv+A zN!Vpc0NE_bcpe9CpRMzn4nd?ULBD=jZ^ndOF+-*@XtG#$7B=zy$Q$yNd(w$(K^B){ zhdKF{9n=hf>pu=?Rsk@d&QtP|17y2+9&~+%07qYhGUOCF_j4@U9KfTuP_uYy*1E&V zj3B_wo?DOhR2YRWe~v%LL@a`=Je2s+q3cj0b_3#rKJYQf2Y`KJrD3%fT9*vng=Xds zhzAhVh$KTG{BFwP`AGnQX{78WwQ}3dG~jr8hJ;cqm2w_CfJ_eps&KaIN-~mYgJAo~ zjpU@SG&69{W`$if$t9TXa_@OD3h8FV8!!>;BDY5$e_*`*c%A8VATtc|TP9GdAjVTOnbgm4XK((io=M(!p;pDn2`Si zt>xm+X)Z;?*1%@Zd0j2>@3lyfG`7O&|Au9?xeusp#!)pRU6!}42pONqA&+};NVa)# z!-JO|cU_}JL^dqxkO8WbQB2N^q&S9D60fr|f~oS#!9qns%BFia4?S@MGSjS~&sG_v zXZirh&H)HV+wcP?crz6n?#^Sss}2vVzzmBeiUlrRq4gnRc>?C|24Io;kT}hOKr{`M zC6Ow9?tpn*U!`nwS4d}@uHn;;X#cerkiCg4%B+71Wf;7uND_n%j>@{2J{=Qy81|Vk^=3u(vf36s2>UrTH=PMH5(o~|BxU;vxx6c>ui zm~i9E8&N={tn1`0{7;pufe zc=^bw@TlO|^j;^-LLyNj6RQ(Vwp1))bJAaD%RE4qbFn3ucmt}?+y6|nW)(&=ws}IO zYqgh%l<_H)2EjW}&7q%HosaGGLjrZzu|Kwh1^2GL-WL)OTsjWB3poQ^7V#p(X!=6b zj|lNY%WGd0f!|Xv^ir6EO~2wcMRM8}?T~u&-j?p%pT) zzZTU0BqJ2tP>}`M$q;NSgU{ETNI}>FG*$H`Bys0-ivV3loFerh#}I^uqd_b{jXmJk zo`5|e*^P`?(BtVq)<3`u&RksIri61&n zCqZ7PUaco|_Iu$}6EH2aaxj zxqm^@BW>(ZGUx|HjwK5gu=2Jt9 zJ*6;Y-T5dNonqCVy16eP{-L{5UgYO;K)49+y`^~@SwTfza(`pg8-92PVaqEcFT1)p4Dgn%y zrDCq(ciVbu<>Duq4FCYu9_fC4-n-e9O2@pN4uEkw@}g0=q(2Ci_%hqs=_%^e6yQ{Y zt0|q zxZl1HLwU|U;yZ}QO=aeviYF?*?&-H9pt!$qj@QdTZA5G1ymeakVIUNTnexfK)x~|M z8*1}>~LwotnDL573E@&@4$*wvsG7GcfKK2@SE@r!}$e2{R~L* zndrAOU~0{x6|%5M|9UvLL@-7$6J=W&ix#$@w3zJ&_U~D8J|i6Gw3<>VsgBO1jJ> zyrp9J`^ERlyuBSa7xD(*j|&=BwBZheX~!ePT}(oPwARnhS;@JLWEY=)ZL77ONaUT1 z)B(n@YaTEN9dPg2LyBWHffygyL@@d4qa)j#t-$a5@z3e&_0XW=kT3;6=M?aWghh^J zvkRXn`rbBdkPEu6K5EvZnh1FG`S+?x|I)aW$Oq3$-T&MZ2!6y*60Os+j-y=TOC%HN zM_M?M&@pwWlR&!5l^HZ2br6ES^_i1Z_Zz$p-;|b0%4U9IC8Q|2OTc z;cY>)7|N}LD_c`#&3&y7_j`)m8_XH2ALK2g-<#?4Pbs17Gzm$zp>JvP{C8G1!0o24 z-&9x$C58CA4p&Ig`vk#W{@{jP)WUHnuPja}H+iVwH1kf)9b-pj9-Es3_27QRlPAr@ z97-Q=I1>i{X8sFq8FNXKuukLeO(HZq8FfBc^g{B2}0G= z25>`#jY4Rx3t$hPVw@?bQ`N-lp8eJZvH=!`O%eGquh8PadC$w7(dKTao~K3)CeMT1 zarD*1bX?s{{o$NIA>*Xbht8oqw#Wu_mcfh0Y(v;^n*GEesl#j6(S(mJsbQ$A%y^v8 zO?v&nBS&sDB>BYO{8S!GKe`W{=bpu!K6ejFMT@=cvrVx#dXlWYx0lZDfeF3UJ5!U^ z(~Yot%s`$P6R`IE5sm;I%LGfZc;;)gJ8Z}K$T-elW96zog&Yv*;NX%d#(f2FHoNze z#525^+bT;zkkf3?r4stSAl2r2V1zYv`h?(_&_*8vAb~A&PgbwnvLuQ7Y(X`@03s^n zcV0WDiFlqC?l(ce3bL03Oa%BTC|N z4?Lo6Plc$mL@VoWbk_gxH#IO2c-g7RAtj=adOYS*@sYMNVfc`te5aHkZhdTcQnNVA zZ&B12{*kO_C0frXlFbziE5r`uyrwkLY@_}kSq!d-(>vXXQ;o{-5>yRe9V+Aln%69H za%K|bjiwC)<*#XEp-}!_JHS02KmLq7irBaO-%muOYM`$`jyMw_01Ma{Yb0lrkWpco zy*fD)u$n19z>sGi=9H!FF4A`WhjOt~BDb>KyyK{kwk}dHx~GDqX%ID-8MMg+JD;CY zJy&rd+!8<%77~npw-t$~mdy$z1jTkl)j#Oir7`)pH2590Yu$l96FVS3kA*RZ*x(RS z(DsbhQllNvsio;6fP3G->U&cWSBU8rYoL9mm2dKrS2YI|+mci8{tGxbA+!#wQGX+v zr7SrOwVJH1IqW`%h1D%N|4>DF3rIlPGugC%Z$X5Ny$Y=c;p)dy$>v|%nUnT_C^~rk zw&nM(CJW`Bu7*a@g=?Hz_(O>X5(q^@-Ysg@-8EhJx6T!VF zdfEJY-|!xIQyu;aJOCs7rit&5KQbU};p@@OV6-81`pUYDw~ix3q{Zk(Fc{D8rxbLp zEXglqWAGAz2z;fgWRWGx9AY3B@beDx{{(Efl^PM{I3B?vp*IyybR}tYACEDBU@I^t z0ZvN+*UFb2g5t+yp?7W3iiO8sOe)_0p$S$_IrNDUIEtt=|| z1DX&l386cvNiIAIdmlXrxqCh|B&i^de6B=}cf|E%p7^%LrJw5j#!x3Zco|-5Wm7c2 zHT^=JjyRA73DuN+I|5R!LxpA#l^j*Fo}fQwwxOG$@sK{K;PB`3Bf2f~IUk2N^lx#Z zm*^Z{>xZ`tTRS{r%o1=l5PSdP1+E1R4}8keML-xXFB>g9%5OHqj>McCNCu^uxC4nl z{Y@0RU#wz#0P+$v;fZI~;1ew$&|!vza?*sJv1&P?djcd0$XZRl^kfi+@uhe44W6qw z>2H8qiUQz+`?=PmzYCR2M3Ei&lOWtS8uxfW7;Ys8B$IsvzO zC(xpO{Yh7nB>Wh9MfgIpbC>epkQjOIX~=uOLE4bxkJQT|EdKp^^8%xzdq4!_1M^vV zbKx{(pbmL}jNJb`)$yOH4S_YeTVe|kS;C-kJTInt-DVYNpW#V}hJ>KV#^8rPw7R$m z$0wvZHOCOGW-xk$69`ZL(a6ta8bk{tNHqJa9$GyXL1Hi|4rHV_#tLIDOiNjyiL5^O zQFSm*2FLsx3Tmbqg+AU&=l&O)4nyD-`jNf2R~XP~h${tjSxEuoKuyHk2>`<4Er3;E z|B$4h-^2$WB`v7?Ie1!H-d4D!M1+4C^XGs7GenMgPv&n ze*iNw1htaI?WqV(k0jopT}`!vHEovM(3YPkKx*LnxvJ>RBjt}zUqe^;B;XLN8N}%& z{uNp&P{4#(7>UX7!#IXubDp&T*l#iIw?3nDFB65AF9LJ4q9EN`4~V^r^en-mZ0IF= zv{idva!mtKFTUN*(T)_nvocmCgiM!Uv;qpQ7s(!Twcu0>%IxafZ2SQr3u z-yT)b4A86K5ZV>yfzEg)P$Vq(5v!%*(K!=HjLgYEu$5d=_A2Ac;e4ehjzn-AdTle~ zOJnrx_dnA!RgeWIrk6JpKqi6A+kOQ5JnEf$-yWU5 zhJCKtbCB-i<6+#DO67n(PC7AcSPW z=vq(=2z|^PdBk1CWX7T3l?@#g?1oLw0SrYqh3*Rkw zK06Z$$In6em(;&|0YR`o%TaD*`*@tJ@mKF_f=r1G*{L3_b}v;YHH+0L!KFYz4pfNdjQ>KkAF5r<<& zT4{Uiiy2afJ`V{{8n9_5YG7XU#71Xu1k~PqphjVNue%1fx)9$!z6q(D9wOpG_DsN| zI4r_&J?w;HK4pe{fWdkI7M_Pb4oMK`x4G%+v`>C!?m0KB+V#L@GwzPu8*+UD*lWY) zBLQtg?*00?V+asuTbR_3i?z)l%R+=SX93&mO@u=!Wxg-X5DeffezSf(Cr+2(vKBEK zq<{kWAlSb6jH_j z2T?H{b>r(@dP8r@Q-qmc_8l46J=;wFv)39I6qv5r=XUZ%r)cIfcSpf_YsT~rG+gdO z{mDlryjajW1aP&Jpi|vSLZD(qfl#+tT!ND7gcD#6(MST0Y*1kIU=DQjkSk_{{T#`w zJC=+S1+;ds%Jsda&FdF0p>k{wzI~5IGXqW(0Ys|8ktvw_BwEk_zJ4FXZhBCtF%PAP z$slMztFddQgIdxS*$MgkeICQ%@Br=Ow)w^@%Di!+#D+=eZnkdahxV%phV9u95|9F7 zD+DNiDY%^-J@kZ{QrrJ3;0kC(afLq|YQDes7Uq#K1l~~(9CgDeH-oB$UQ7SzD6tb|DTZB?0Y=8?2UV)wDRht7ZV_iZgRi1ZlL zyjA8Qr5P-?64&gm2OSlXAEl00k9~kjP9eC!{ng{QwH0up_Ny$>DWWA2leE$1^q{T4 z|2bGtA1tw!`UYIu*7uJl@JLTz)xw++Lqt&ELP z-d=#Rls-eLrn&d5rCt0)Wgp=Gt$f}d*QNB~=h}hFg%YoaH4tl9|KfHkB%@|6eEjYy zpxV8NZ~?8H1w;c9oPwSu35au~_|W1c=^$|?p9qx_5vdPbg(@d%1JZ}<|CmulVsl8> z7{7oN6n?pIEyaYE(IJ4(OM#Y>2VJI~# zPgq9P*PUVrd3sB>0nF-#;@8|PYGv(bGr@09d`U#Gv+%-o9sy>D25)zWnNN+~BuINs zYkvVsAqD<-FEA3Y`22}7Xqt$PE4OLsBfx_~43&Y&cLb757OZIQRT8NHN^1}7MpS*L z#o8-UfBGuWXcRsMW9`CDV~|Lg$8#E^);v5$tBvgg)^&cAK2!@-%J1=V|5oG7a3FIo z*JYz=a2GKeYw~AL(_3z#5bI?8%{zOJ`ioJ(8~z_th|Jq$b)6yh0twI1FQ|bafDLIMaD+qb?G17l z!s-TT1PwH!7=?9f^H%E?cuvyGRD>!_fByo$R>?1~eH3$ca@^;1L|XYt)UuCYlpz?d zsxO7@wwH4@5$iOAZ-rig&tA<*PX{YFULg7w%=4Kh3!A$I8$k&^x`&6|M=AmTB}Xrc z(cXuiMZ&Z@za=+%A>`rIBwmlL`-rfUs-lm`&Cn=V%1?+z5+P_u>PF^%?M$YDV5;yK zo_$s*BGPn}JdZ(2ft8kCX@HCbN4*`HrN;EIT5yx?>o$~*nY^AVn9fp(Pt?4wZ0xBXCMbrCkUn5mgxZ7$Ers42Rc zSo!I-w%GP_)X!Wea-~oldllG_p&m@CRy`Hp;nQgoe@Au)Y9WRKP&cUE+E|jTeY!Lq z*OQ3`uoF5GXMC);x`ZaEB{9*vwXNiA5&^>{*4FG7h$(1y>5)C&qLf0$4{CFUxE&X>KH%hi zfk3yjB)pjCwl!xawBNk!&e;r-r>E*X7x#k>MZb#upqKO%c6Yl|30y^7?7tVIbd*^2 z2qE-B62U&Ld_KwRTGv`H;5**k%-NQwKcTqh-M8PsN-d&7cWNq?X`B!JHA6wai zF<7U{_Uuk;byivMY6;f&;K+MglAb<^*oS;db?ZQ>>!i;E zr#tk_KIWLU4;XcH*73l*dQ%d1^zL7265B>ze^e8ufbJciGyy{vL(&lbY)nQ5CUh(K z1l}?ba&J*Z*EN1Tx-|k5_jz+4fDy{Zlk`*6Zsj)4GhLCCtY?j~&QAV5#ENT8hD<7UC&6$o3_J`~DnZ}JvO%g( z&bmtAB0v12$W*>?R7%l~0h3VU-Ce!)ttjV5Aq(8IteZlD zXK+1>Tx?O;Z}FChMJ$JjpM9%O8wCwoPVJ1w8$PApU`4ieJ!+LFE#y1CO43Fnk8z?$ zz{8VQ&uh!hh*<3vj-jv1B~<_9x!kZa_ah$c9m>S#q_%95rd~zBE1mv?(FcpGntgc` zDaQo4#I0RI51m}quunf&(N!14uztVc@u z`lM>0*L3KJUE8%HWFn&trLNXHw2<+z)ixoe>pw2j>lhWFulpVI-ld@y8T;$H@V3%iK{BSSND|-*slk#D$h?p;9iApOvTUr1u33y1BLblkV&i zD~bZ5;3T$+wm0%QO)s`4OzCq>MGzgkD8<`2w03Dv;7qxw%GvU3{88o#-%%=>Ydm;@ z##CJ>8$}jQVt;~!gf#c-H>crr;u^>3+7_Z5mPYmc^BW$#10BlQn;)Nz*hf;7lQPXs z*iPl^V8@TYDPAD;;d!|u&B!$+NgRQ(L4W!lAo@I%Ck@OU-V%Gi?jllS*EmDJ|CVHx zzLCjCx9Y1R#irC3kyT@k3~A>B=j)rS5|=Qqu26({$w{|#J?<|7v|W@6GX&M7lCO`h z=9E%KV#BYJHfQ>joiL-q9W+JzK8s2jRy@XT*7g(c`|f?}k;W~%BLhcs+1Q`#((l+& zs9yg#Vb7TVCx_7_H_Vh$i((=~>R1w~>Vpi7!ER-THPfrMiYq>`kD^j)JOJl1cTi}d zViJbF?lmo;B#E{saFoj-<$kf-=N#8NFHpUcDqolc#?EMKP06V;G7UA&GD1JWC*e^p z`O0RE#9zt_FRZhC@@+e%3#UWpM2HvDnm)>GqbB}nV25CskNO5lRi^ECngIiai#(2( z1OB=CMtYt2y<TPRK}}do7^R15Y)vT=C5HjIEQ4zBjkWJq(SFcEPgysT z7g8>sH7of$ijmQuA!L)F=XUhgTKJwb#I!uCEJMRM%Jh&qiVUV7y665gmK#RGJ4*wh zbZUj2@xM(wBNfpW*^koE&3yYB$n+i8BdiV6?wlAo$rx3kfLV$p<5~|`qOCY?w!t;k zt=h`Q-*$A(NMq+pRLa6>g@J`e$mgINMq=Td$sE4rpLm>4n5l{?FJA2CcQyvfSsS0} z+Zr2cH{|G*Lcg+cInz3Ujzy{y;5TtL*&C@z!EtLLHNjCi<2ZVuyQAq9Q33-;Ts}JN z8qQLXijk!_fexrJ=v(09snesBw}cs_r^bk21l4u#dn!5%wpaICq0yxd4D@mgQ%&d zct6vrVv3*Zo77gpYs=b!V>rtCD{)Dl5#jINzw+ziQaKGECgru|cZvb{wtMvqj=^Qu zCA#sYWYM94>oPe^t*>9I3R@YK>!EbDd=wa?uxC4nCzz514@Mon(qwUlkuqr#G-etN z&-plqqHejlXk1WhbMfofoZwxW=N4<72^aKS`trIVJgHXgIgzh{pL1|fg8k5IA%o-* zSuX#`dSUZi=2+yEJK$9Gn8>A!o2ZBxLeJ6ei_;Nt@=GM#eRiLsT&e2;#=y_|V)hOh zZ!K0=3n-P`yz$=OjUx4K!Q3haVExQY&KOK7lk^p5Wvo?h>Zpke;nPcf+7e6N7TPgD zWv0dHw}B&(ePxrZ5%+wgekEbSWxi<>&7`hkYQrv56pG@@i>ai@%Dpo={rw1!h-WKF zVvwuvQbm{LyHj`btUq3<1Uya+!>HF8@w?D9k4#xVDKq@;6J{dmP3EvGRXeyp(8p23 zJc;m4RPR+J8=W&wgic^6)b6NYxIdNFXgRf<#Z+cuy5 zgpV?Z6Uy$w`_Se!85Lo4PJ1f0m6WzqV+hl*aO>D(;?-tTZMxT z`y^hv34C9|y`bW<`iM?Q+ny>%uFHPYAoSN}RP#_Fb3tv=w>@fM!`Upv`>vX|2d;5k z7>^Fao_#cH1kKXXjzOL29i2x9?y%mZ_x>J|1a)|E*l#56?2Twu*Vp@;Cv1Wl_TM>p z?vrVlOEV_O6!GPSl?CeMsu*Mt;YTpvJ~en@R8T(n=|}h-RGpAg9`Qy9cNpfRmlec~ zIpa%bqEja7>6%QHNtL~~`ZdOb%C)cGCcy|hEmdnKq?h{UuYMe8>BodoZF%24NO(x= zNc8g?>79zHP%cMZwv@&nY06r5Z|MS!EM@PMQ?==(bhahvFgti=H4L9P4QZ8SzF1>4 zUE5d zS_I~Oe`7lNZ%nS)&om7xk@FjTePP9wL?3_NFH4!^9c|mUwcHSwu2ifO!#fn(M}VSz z^L&^``s4ChhE>5O(#+P9MSViji-_x`H8*F^cm2dCGLLY&qV!sHS1{fJ)3e%XXM>kD zdy}7p%Mjh54ZtWpfOR6tXd9s&rgff4(oA;G9vTrzFu1NAS$2NP*LJ;J>H|&G_F`*l zqRdlwq6@^H!W%*Sd~h-e?DlHPVg)U~(tLCr zBy$aQ#CPRV|56cRHyd;^ZZf3yRhsB|bHy<76hlT^Uc&c^+6%dw>g9d9A|9bA*8$EY zx&ewdsd5=_CMP`6dWXyWo)zRaf*qBb9U2C}Si0cufwGlM)VqC>`Qgn;l__k(-TBQ~CIiUG<0?L4- zwr#!2JevHjG7xLFH<10*Dz~7TWv=c_Q$h$4yug0TzZK8?B83d^|q1m3j@N2!$VSEfN;zE0$< zn@SdtG~Lu5DL%m&Cp2Izs8nN^rrP>5(qHEC>ks9M6UKVA_v)VaUoVJC`K_RJTA;19 zo3f+e=rc~e&6Zt0vl|$*HWDiNd|HZ}k?bsj^z>A$0d;Zmj3E^dTfq1Q5}g!T6@Bc+ z)1${3<7M<1H3?OvU)dVT*|WW(XLMS$P$_sa7OBcR=Z}PqNJp)+#P)Ew(5H%ci?^hP zme~x{D^Jr4eGT1JzD0bgKFX;UM=7Nu%n@)hbPU5c{Md0YQbdQ>DMb2+mTC73qP-s~ z_T)Rx#GM~`yLnkDn73ZD(W^L&T%}7151*J*^brQ32_<6!B{MZ!Le|^d#FZ9)I<4q+ z3PzMChxe4vLN<|r1pPjm$%j{i8J~7Xfi_mUEf7nn_T`AHSBoG8Z6RTuiMcn$4999S?R&Rt5@(-lej#bOB4rV`;x zwu$}XVA5(w-8k*qHKMai)y}*fddHggXjo#k7j&{z>}NJ>yQ#Uc)*8PaLU-m(YZQ9w z$&r(d(&YG9(TaLsiO&lKbuL){;o_N`MXb)KOwBm&n@&n%KnqaK0l zkmm&YMzb2iKcuM zUEhIHhY51;w?J7vWBJeRggQ=J&7J2S5zv7|E-0Q>KPUd3_5~UL3_p``Bey zlKI@hBZPNy^n^QVja&x1`bCt_`=|b+{x+r!SLm@|3$g?XdW7tan9L z>(Bxquhj*A%7Rkp+{yjgQ;xmL`wX#D4FL047C@wX)-oT(>x>^VdQL#GGNbYUS~5fm z=t~`UCF2k z=UJ;5nowRUc-*mOD`9qd6rt+yBZ&HfXcB1LK44B(cB~^NFDPRljnsqb^9sdl&(l@d z+o0G8M%w|e-ZP+Q^YZFFsHu~jVt3k&FH4js`Q7HR&k-@HVE>m*M#@$am%;gWD_>y{ z47!E-Un-rHPteFN&kUk}?b*q9IrQ+pwFp;D@x>8-hGK$T5tJ#|#kEI{aN^-_o-4lg z+^*((`68jE;7&m=IBx}dyz#zBSv0aE(=L7>@Aoa`8Rw7t567zhS=dpzeNFFMq^+%W z#GSudk~y+tZ607Gd3q1=ix&q&-A2S1dT}xfkzaSbJ>m9Q6pf;8_@*Yve{crW*;YG- z3q$8?y);=`-++-mSlMEP*AV;?VH*(YE?^=pT_q`A_y`{Al6w{}ccOnUOX{rF8&i!3 zJ#iS{4*2nC>EX$Ng&IVtIKSX;SpVk1%%I3C-s|{JEOdM@bvpq9(+6~Y{?*TwRIox& zm>NbfrZ-utw!abt*TB<&-2K!0Yq!CaRv$WDiIZE(dh=V37(3^__5|M|-R||ARd5`S z3=Y8YBOsE5!z(*P!*;Qkz|g=TWWohC+x>(c5mN`~wjfYTy8-nY)LL_dc7CEnMis+} z6k+#sMJEayTtv@dhfSVVFB5LR?&`^pWwV$}5su%`YUS+MnUy`!>eWJF0#z%Aq!Lox z2mMr;l4xW>^yA(&cstMw87w}p!FEm{YAAT{5 zkR9-eqC2qVb|*;>c6ZjOCkgaf?^WHaa||llGNRMZoZir#LJS=L*_7)0L>d`TDmgYM zEIN2_-x7si?V6`6h7uM-P`Ow*MHEu>^Nd%!XP(nVe0IJ8Nyy+|$>0Dx#1W=^5cYP% zns))NF$H?!H9y}0KOP2A%iU8>P%C+-DWg93C-i?4UgPqMg-$aY2LM=I|K4~$nsMT& zqb!sFeEV4T*g<7`by5mcyjH&W-M6r1j+RsIWjpfK9>@=+G44#}e;$#$eShiCwDsfd z|G4a~X=&~T*+pQPX%TxLc$wDp`;n2ZFw*AGN-TIASQ>&0ke7L?zQoZf^D>Snw-D{2 z=3X#T=g;FaMf`XH&x{au$%wsE(fCaO-W=Y5mYc9M;ct?3Jp4OC9PQ?=Al9-Vwzg_` z9JF852msk)hZ2y3bA4$hzR#von!s8`;1#Po{w;ARAlkjazaeyLh_pb=b-(X;w(DzC zt&97>f9ZjOmmrUd<=xVB028oBF&ZAw3CpWs0xDF7Rgv~WU?J9nl;CZ&J;YWB8h}tS zweeu1uhQ+4tWKxQ`n0c!>JI{_gXg7gNS?VN{p|BJ@KL7DM1C!tDDH?ONBJ?4yYy%g zU41DOy8!$st65jz(1MJA_llS4fxc;2{cE#&ZWT0Q0cU}@-aaKL3dRUhC60rpG!JH1 z(Pt>RDx(_bW4H@8T32&geL6G;YIIwJs zzlfuUF5C`^>QZLLq9QM+eZT%mpnm#gv|4fAKLj%|^-&Ai`JZg7rYJs}yU!d~2|EHq z6fB(zZQuW+$!Wa_<)g*d(E}Eb)dEr1aG~Thb1?PKG&}#(nt{Jfl_h$h-l_h9`k@N` zWUsB-pM#sN{!v4Wajn2SmMq*}8ZnEvHL2)pxN>#r{S#&q{Sq*gqafV?R-R2T#J>+H zP2rufn~9CkqjB+?BZz6$a7Dm%UL^`8oba5mOQxv$0kuhWe!2wzM+;!|BgH7=g#O#C;cM$NW6<7e1Ew_ft6k5=^;17?z|LpVWP?p! zgWCE$KTVMSbJ;PLy)XW^{svD_YWNUL8}0kb-@q`Q`lKG#1r`o4IRE~5>sGVjHk2c<$|;Vs47 zj(6fHR5szXx6b3mT>~k2%oXUW+renohNEti2Z7PaiM16UL(p52g^_)9q{so5!#`N+ zyTsYb!T_yCWe~zyrUwT*&2%b6IJ)6$UuQ<@obvw_^n35JCicgl(FUe++mZ^psS*NJ zkelT59np{VhtjW7-OT8-bCa0v}Geqa*}d$HIK%8Z3r@% z3_Q2syA!k%wEbz-f7^fJ=d03VT@4>!WWD@-ZNKbKVA&71>Q9&Eq5jAn9hg?NvGVy8 zyfp6@eA(MYY#nbJlAi|djMs48=;oUTj?_&K<8=N@#cM)ORG;9^Gint?ybv!Ng)UY<_n)BT$K}BKN*cNk^At0 z2!*cDtpBG~dAZ8sxb@G>+o72je|F>rd)Vb8+T1kUFm7$C*PO3Irycp5^@^W6)!fWG zXB2)0G3LuK2~np}o{oGWFw}GHeNPg(%OxN;vu6BWjHG`xUc8*;d+99rHDz`sGgC2y z>qe?>z17Zaw$+e83;)_b@7)7kEU=j#^{(%~H59O|fKQ;tWNk&sKaSC?nuVvHJuhN&^Ooa} zx|i|@Z-2&0Mk$``5zaFvw7aISaYtnff7@~L9vu95=W)j+vT23sFeX<2wDB6_xD%aM z)2d__KcA}DlrMFnzZ7867Ki8PE8TqUmXP_)&a0`n;`5hG!jGM%i(|SOX3^%>OAEcb z>^MR-6pvgI2!7VyiBxBLM#Wvn$9=qc#jTVqWHh*}YEbg&k>rg~TBX8IUl}P!Kr}Hz z@8fHlib_77iDo}{ckT&S>Iv4aPdlkt8jTkvxh_s09~7P%^BOa7vJ~+ zrPRv)Uv7P1(B*n}5cucUj>EGts013zCFa8QOtFlNt!|uld3&s&DLC%M=76QWCZW*v zbF{&X`dqq!%%T=_C$=h7(6k$#hCRoVz>I5ce|uOn6Wc*n@q;|~SgKltn=eoF%;1-m7fflM?%gVG zq?5;7=ileYmo{$nURuwg{dp&FEe1{+Z?A`T>1mQT>hFiet*(?bYHT&?w`h6FG0gO9P5)k^^S{L%u(b8lN&aY)Gh*M-ni4?Jq8 zBfbo{*fV-A|Bj*@(|V`YBSbGx?nR#uHOP(8+PNu;74PJ(vdA;OY~@#CJMvBE|CDv! zaZN1k9!GjN6vNTbYiI#PI?`L{0S-l~NKkr{Dn+WH2uPJKO{z!{2!tXKdQ%9}iPDrV zASg}l;yL%e_kJ$_EJ=3enR#|*cji0${06l>@Q#gQUQG?Tj|YNu+GmKRb@P~G%Us{- zS)y)o6^m=ptJ?J9a}9%z*3q`Fx@P7(>v^bFSw-r} z{=U4|cz(3{o`ri${g>fmSd}~4X57rbh1E_WA<3p}siysS&T;Zp*k-edIi$RnUE6A0 zp6LF%t-E`UAY?B{mENgar;EgOl4+&tjl)mpHaJVX&Kv;pMhOrqcF6?shKnt_V_@-6 z>(vUqG-BE7PXxl{5+L2;ehXIQeuX17SuPseMmvdHDW(d$&x^i!e*Mebt%|9N-BZg7 z8-d%H8a|7J&Td4VdA9O`Ub*()u5c_;xkbmlX`Re@3E?wD(_eG|drJvxq}kp6HerO3 zf${UsEY^%Ivu9Lg`toR7&vTGh$Ic|abE6v;r9>1}&N!fY#NnE33xFyq(~B}2HHyKT z>!~V7Qny;-tXE!L{6)xViki=vFQnDqNomr7kYq>_*W7)SvtJlOIE+|-Rx0LB+Vc6+ z0wDLhn8#!`yyZ=b7~21M*&%U}O4cR+;EmYjC$QRJUrfC4$SIdl1m?|jF3h^TyHL9K z7k&5rLy0`S33KL$c!Q!l1fvK>9yN-2Cbp>22S>75v|DQ76giEtMrgx3+Lp_%OsNx` zk7N@t-Gp%_D7SjcT2G%#X&K{rQ74koU0~+&JZF6DlhU2{X`^F{`pvxBbZmAh3_Uo> z3DE*SP^lp+F$rY*f_fITEjP&f&5EU*m70#$Kj*U4>Q~-Gi3fEG?#a{?F`1cmQS3-W zqDhp?--^Ye|G3P48mok*$Zl;b%uZBETP?O(_AFuxOq$`VL1OSWYW!*9&L1E57dM7W z$B9Cy>P#7J90ugFyfzsTA?^L5PM9Wy-Ax}g%;I&q?b^QQEatGmbJ3iYhwO+TT+dLe zf%Un~tJYu{1$T`HjLd}n?AnEtz4%CImQ=Uuhh2yy^(WBE{4c5lhMdEV(}|LVUay}& zxwVNvEa}kRsH);QWSrTuf#10d7-v%>g^lts7t%&owMw=|XXS)XL@f+dvFOkT9_tLd zIe0nmX~{E%eghYWLR8L$n%(J1kI`Qg^<@u0d9cwi0XK1i9KYrBb+ZQbl8fBXX%r;OR-W+MH`7qImr|pf8A&spjoXWPa!!Fx!~k0 zr#ocM-1L?<2NpX)P0v#XLR%(-Q(@7Aw35|Kh`32gF!8l7Zj&urayCLkeq?l)MrHn# zqxv!mpXFTdRg%rlhnd7R#a7h5#HqzXBI`L{VM0`0bB8Jc6++hqAsEOt=Oy&0GVuogIuk-)II4GhrOorUl|wuDJ3G_nV*1t1sKk<*Ir_s9gBu}h_;xWpPf3!I zz%C&|sjXbjtISlSXmyMOLPrU;|0ubpCTd^bpym1O*hqnkR8aroF?_!9IohE2Hf8>e zA9&u*8f(L`f1j`?Sr)M$DJ$?8$W7 ziV0NR5#dU)Z1mTo_Rk>cg}gVGp1<^Juvj2Yzn+tE`rrU89$_{0)5$Ouy;knPSoO<9 zC(V+b{f6)mXELO2`Sz{jAcFXIS=qblUEh2$zWt#JRf7+gg9wNTydL2#ibAl(GACNG zQO!*dKaQ8XK~F<>KzM!=-c|0A5PkiU=iYafSZeoHChhB(4Gl^8zStLSs;4b$)u%d> zSDL>Ub!Icm{+bu0`uoB3{MKh>;xw*DgzqsWETzvA5bi-MS@UAL!Nt$Xe`>wmx)zhg z=GZuyIT6gf+xiKORsScor}LzZt)>dj?N)Ook55i8*K{ki=)d+rgG`X@~j*b(O}GQ@-A{ zNNxLU00UD-=#K%fH8%6o4p#8-EhRG(Yl2$T)6P_$x=p{!Z!rGMyP60pzW>usOp+f_ z)fR8GjsvMYx>ib?g|)VKl+xF$N6TfES$zpIW{n)9kO}p<_GUR3YdX8(=|+7D+DDW1 zpbv5=ilV+}S3`s6dI;l#eWVas*sfT9CDAW2wbTuugbCWE$Xq8KH5ImfH@Q?EB;xXs zZORPQJgVzafh*4A=;5Z(nU=Rm5w{1QSntG5TPxq1FW+~hH;(wDmi9)DSgdR!il5)C zLHK$(BiQ`sOzIsk>6j z<2cA#IxuO`ALbu4a4D2=0j-lMwo^3g>qp0?bhvj&jAt8Hi!qkWyyR4o+@OOMdMzi% zG!DE>@*TKOUIGYJNK!oTdii^fM-OP6yxOwN>LmAJQ!yx)sQfFb{$*vp-w)l79k!TLh1u||= zeYBVr`LHDMI{PggRW^XJvKe) z>n{6_y_b4+W4aEc*OeM`FfN?n0fp8n+VNzL)F)Ln-?Xxex|P zxrG@T6+79?dDoEc62W61WN<l4sG0VB$_|sQ zIZH~-vTt%_t-da)_37G5RY+qWH}DWRzTnn=z^T3BnuZh>iA-+WVx-IbT<;!61{tDYqC@PcKYRu7LF+5!*GE{TRCgl zVgHuEW_*q7b>xP}md|oiuB7i^f?_br(g<|Kem2vu@X78)201a|!UVZQ zrtBIt;fX49(F{-7N>Gam@3r2luwM<{AH~@6jMCOJY&%>JTA9-S{$$}&hyU%hU%GbQ z-0^nd$EDPe$f;M$+ECxwp0PV-Ntn$(FX|L|ll~e|5a%PJ&1Vl@0Gr%l$)zpU!(4QH z?YzE&IJ(L@Y}W!BpDK1Z^S$^4=nn8I`5Ez@^JLL3cyRMRl4TI$cYoc1^@J~Fjl&@1 z4ZKSp?+2e$S-$HcA&k^SX=lIQK?A~l^Fzw+mZ1(t#Qh9G=0Y@ zRtXXw`sWne|8vSoZg@_^Jny%Ooq<2+*!=Ho+_{p~oU)JOAsJiT8-6+SETOzQ_~DBT z_vS1$`9u9brq+QH-=XOPUefmTslF?;=q`@u_<9PNyUJ(u2t0bgM;QA)Sd%?=;?CVy zJE1zw^TBMe$SqsURgM07tNVf+sQLSnD@#gLQ@c{O7VcpfmkEgUuoQ+I_a_(F@k!G^ z^(F-BO=L*(GiMJLviAg^6Dv6&pD9M%eN`l6_5B9fD<6tnmFU%MNTmY+j7RxM58lnU zF>~b!j$ZgJc=fL}FUoRdUjN74fk0!pRiqbOzY0jf=r+%T%#kTZXgsL&yvg!7=8w&F z1>my9JW?$XlKK3UC`Ky_P9L5_C%sXUIs|A%WPAR}D4Bf3i#P>J^YQ6|2LK>8K>Xrx zPET>dtC(C1K#-Ur@lRg#;WCLz^rbROPc6>>{Q@|=lmU{CGB;)Ua3)_lLkT$i$!>I? z)CwFI_KD|G=&M!eMOvk6!>$1GOLjB*ZQ|X%+=D@?P+6YEr5q6blO?KQ4a`EyZi=zKU}nMZ=lmycFo%iA(VZq6@T*bTf2{~{0Cmd%y$=o^1bp^x z^)E--Ki0Vk#Up`oW_3yU>^uejp|wK-fwVxwqYt-G2d-pwcb>>rH1NYV)4%7Z#sQz; zZdpQR|EYtJFHkSxBi;c*Zo)%IIGMut@U>i@G5*f;HId;yAkslM;!DPH+1(wdwKb8j zz`(P4yuxD^YrnI(>5ithy^uXYk$^6$m{4~Zk34{FhKkuPzSRTU8DtuSY#KM*`@yHM zAG_vLaAqZQI3hkQIqxc7wKtYI%5vq+yr!WP7?QZ-d$b56TkDb6DgY1Ls?p1F$UNUq z{J^#W$iU-<0&CWNl%~)c{n+TV;s)FRh%Mbu8lLVrZuABk*Ar8L{TE5WCDl!H&TIQg zVdn{eq-W>p-!1ryT{B5Q6w3Q}_F<;w;-+cdQp-xB9N1x54u7J*FH?#f1?uf0+`2u* z7dufu{o0~fjO%-5=XoiDsN?t1TPYbXO^9o)^2uJt|#xw6hwEc%T|9TKa(#BzC6Q=e9iN>nRQ6jlkMC~(SbR2 zhzBM;StuTS5Pie+TW&wMBG8{wEcu2>fgUq^oj%+N^qoKDnZl34wP%Xw_h(%jMu*Pu zm%k5cpWSu+$RM@&*tj@q8`vml&A;=^x-c>Lbh1ESm+!Cecvx#Wq9*?^_iI8P^a4oY z?yT?T0ElaNtG+5YTlks*taS!CnAjXs=&+wnf@ZbpyC;D zK^W_^_Rt`pW5T$_3y}2T&24MKYe#RxLW!mQTf5r&yMD5V6jd8LLUHi2W5&E6$^*?KV64@bTf5$ex!}dUSGsf ztJS&zb4}HgA+wGNyThks@~=*0Hos4I{%#EZeOMp@6h&?r2&u8nunj$__x#cg5Z7mE zSDufr-()IYT^!=OUbzTFO3fyl_*#Zt95D&T)yH0(Y?^i6gNo|5G2H$E1Q&G%Nf?Qcm-U(L)m~q^5_9xy zHnBkKts;fs<6W!Z%U*3;e2ORVlX?7a)7{5qauOHI>LNji!;U$eE#wi`9x4`te}w^{ zKBT@+cm#Q#@KC;ZBeV9&QI+kYydSWnkkLacSKYPJ+=}2$nOgSIMeoI`=MfvnD+4M_T3I`9>guSjI2o% zx$fo>0qX3Eu0)8M~!T03*GexWXlaYt87%VGIVa`q2ifEWL`(O+SIQBRU4P;It^p8 zC=r%m2TN}g9`oIx2#9)wKym+fG!^-#r}&%n@}1xLM36YrkP#jssxbR6d2LC z+4Jj_eZ~CSAt zskQIa#EpJ{9P=wWUtf0F#;*66tGFjCArrien+2EocoVLOG)>omv3!OW4~}2S80@-;3pJEGQx>1 zS4gX>HUHJ}QYcgiWz=4)83em6W(KSl7?~$?!UQ#sW>Y{s0<`qTIh`CgHr)&D-HIIc z)tzTC?>)dmmozmJ;x+0~(cUp_QC>T;_Lo1(6x$JS3o~ZG))OqS z6lE1ie$O1;O{X4J=cbiHLt085Ta-NL0(5Z~D{Lc&!5r>2$NzWL;NJxHazfN#R1Mf6 zdR`}2f1Wrbez=^egIyb(B)dM)Kw3qTEnt#Uj|fLUEf*D9(tBPQg#(N6`K{VN>gN2A zd5ZnJCnE~##SuqRA&iT;l`4r)Sr$t9&48cmEf3J5p?_a)@SP5$#(%$^k%w68988Ta z4eBk5rexeVS5AXQE+@>bhk%2I46#D$w@OSsrF^5R^v@7(uf-hYV-RS6o8B$HJU?L|1yGDUO^NnO%y0RGPI{BJ|sH<_7Ot<(Nac zKh7vWo#V9`F{kRiZBiVH6*vXLCzRWrLND{xoI;x~)o`Iku_?V%`ixmh0N$sYcx8L{<<_HV3?`kfU@86(J#Z0%LGn3j9%;R!kBvDy0WNJXjKiMv{)m{4(d> zhJlt2K7)%S$aVb%^Z>?OYI6xHO^PTYjg?yAjebr<<|c01{YKLI#2Pnv#EcybQ0ku3 zkWoDV2KJjYhi3o_KEzz`6&*jcJ{|WSLmD%cxqY%O6|y=6HxJi+%`;XA?832p|DW-A zYwrIak562U$MZsca&_FccH16U_+Z}Tkc`nTso1>$WG!N@!B9rJb2|?^PDVW>!-r5R z*kW$P9GU|N%MtO1i!Ag)5V7yl*=ck6?CAWI*JA6(Z-bN2oanrUne#(64|AU3~J zVy4@7r-(JEmcA};APLbe+34t1y-_35lD9iGszF$K8Hw0r>f0rxfnS>52%{v{KrQ)W zpPAx`rQUcW2JPtL%8i5eLVsd%TqB^qU1C7!#-axgD+OH%^9ETXNyY1Y&L0;e=;hZ5 zzZyexZfLX0U-?Jxzx~s2Oa$_@*Ajiq61=cShb3!S-mNo471qYTVM;lj^}OcAhk9n8 zs*#2zw^%*1z7FcgyX=*#+7b|u+Q8M6O^zP5%2F3YX}oEs{*o2gyRA{yZ-@8Giav7z zAyiaAprP1Q?;SMD9wYhokL&{(KSdNYx(G)*vpXGzrQLXlW85^A^8}=zYG83jB1Kre zfsir}IewXHitn0~BoS0i>v1b7Rr8^Y%#l!;U+lD0@S=A{@f&sFR@!-Wp>c zZxUwY^XF=*assxyQ!*p&E{eW0<#_*VV%^iERH(XWvkP5vG+KjwN!atRting0wni*L z6|yEu)AZ#%>_yyrp716E=&7E~f-Wx6kco_r2PVU2UKe%8E%NQFBDH~c6tH>@zp44gH#?wb{S2<6f5nu_?TFHge#&p^)fgb6{~ILZt)C-jMD$7=x6*;*=W-MvGof6YRPMZyus6MwT|ILV`( zBHJ$$u>fE3Cyp14JMI4azCF{g&5~r6H&lNuHZVv!)kv+;qiSsQ;S}BA2IPIZnQf1O z7=rmw*9dI6~Y&(~0p9w$kQ`8U{Dy(U0=Y1$z zZn34PQ{n8^+q_3Y^?HjPs$HW6;!;yKGQh&A8NI=)ND$)mGnwKHs7luJ?wrhW2s#Hl zI51h1h(+Y2O^!}BB3OH=r+y;!ren`~p@ABBpM(PE(#6j)x@ZmMf+>R+X-!nOVY83_ zCNX-*0P=sVZqd3tv)2umW|v$jT!Qf3Ns`?xkNMUj`Tk`Qu#p!>2~KadfA2mjS6UgV zk)5kF^b?wP)_GrH7YrxPRQ6I1Kikgt{@L{F$_DRFXEX!&EiP z`lix`dRuQ8y!hgX2wOS5XrZc4mF3FH4otII?|7Sv0dORPmJsX#V=Ia0l&JJ#;}rW{ zY)^aoX})}K@aq+m698CCy%_*Sc#l|1OKEk%c4J(iF6@qZRm}|_FX-R(ArGER2j30> z0Lo(kml^qw9K?!UVX={?j(!Z01kFhV4|eZ`f3+`Yy*14#A<&V>*CZhP-!!n{nQuqM z^(zxDLc`>q7>Zkd4-GEpmhW$EA3x#~Br%&V$h~=dqMHd2CY=}n*6{R*6tEHj3Hk`^ z9mPJ9M^V*GJ2h=kvKAE4iP{#dW`7%U{6Zg&qy_-OuzBQI?KKKVA0f%o_cxn6Azwqdv17b+w zfAH58pOPXrLTtvEz5ZkGpOk@7?{oRI56j=wSN Any Fenix function without a return type, e.g. #Fenix_Init, may be +> implemented via macros, in which case it cannot be used to resolve +> function pointers. diff --git a/doc/markdown/ProcessRecovery.md b/doc/markdown/ProcessRecovery.md new file mode 100644 index 0000000..56f338a --- /dev/null +++ b/doc/markdown/ProcessRecovery.md @@ -0,0 +1,116 @@ +Process recovery within Fenix can be broken down into three steps: detection, +communicator recovery, and application recovery. + +--- + +## Detecting Failures + +Fenix is built on top of ULFM MPI, so specific fault detection mechanisms and +options can be found in the [ULFM +documentation](https://docs.open-mpi.org/en/v5.0.x/features/ulfm.html#). At a +high level, this means that Fenix will detect failures when an MPI function +call is made which involves a failed rank. Detection is not collectively +consistent, meaning some ranks may fail to complete a collective while other +ranks finish successfully. Once a failure is detected, Fenix will 'revoke' the +communicator that the failed operation was using and the top-level communicator +output by #Fenix_Init (these communicators are usually the same). The +revocation is permanent, and means that all future operations on the +communicator by any rank will fail. This allows knowledge of the failed rank to +be propagated to all ranks in the communicator, even if some ranks would never +have directly communicated with the failed rank. + +Since failures can only be detected during MPI function calls, applications with +long periods of communication-free computation will experience delays in beginning +recovery. Such applications may benefit from inserting periodic calls to +#Fenix_Process_detect_failures to allow ranks to participate in global recovery +operations with less delay. + +Fenix will only detect and respond to failures that occur on the communicator +provided by #Fenix_Init or any communicators derived from it. Faults on other +communicators will, by default, abort the application. Note that having +multiple derived communicators is not currently recommended, and may lead to +deadlock. In fact, even one derived communicator may lead to deadlock if not +used carefully. If you have a use case that requires multiple communicators, +please contact us about your use case -- we can provide guidance and may be +able to update Fenix to support it. + +**Advanced:** Applications may wish to handle some failures themselves - either +ignoring them or implementing custom recovery logic in certain code regions. +This is not generally recommended. Significant care must be taken to ensure +that the application does not attempt to enter two incompatible recovery steps. +However, if you wish to do this, you can include "fenix_ext.h" and manually set +`fenix.ignore_errs` to a non-zero value. This will cause Fenix's error handler +to simply return any errors it encounters as the exit code of the application's +MPI function call. Alternatively, applications may temporarily replace the +communicator's error handler to avoid Fenix recovery. If you have a use case +that would benefit from this, you can contact us for guidance and/or to request +some specific error handling features. + +--- + +## Communicator Recovery + +Once a failure has been detected, Fenix will begin the collective process of +rebuilding the resilient communicator provided by #Fenix_Init. There are two +ways to rebuild: replacing failed ranks with spares, or shrinking the +communicator to exclude the failed ranks. If there are any spares available, +Fenix will use those to replace the failed ranks and maintain the original +communicator size and guarantee that surviving processes keep the same rank ID. +If there are not enough spares, some processes may have a different rank ID on +the new communicator, and Fenix will warn the user about this by setting the +error code for #Fenix_Init to #FENIX_WARNING_SPARE_RANKS_DEPLETED. + +**Advanced:** Communicator recovery is collective, blocking, and not +interruptable. ULFM exposes some functions (e.g. MPIX_Comm_agree, +MPIX_Comm_shrink) that are also not interrupable -- meaning they will continue +despite any failures or revocations. If multiple collective, non-interruptable +operations are started by different ranks in different orders, the application +will deadlock. This is similar to what would happen if a non-resilient +application called multiple collectives (e.g. `MPI_Allreduce`) in different +orders. However, the preemptive and inconsistent nature of failure recovery +makes it more complex to reason about ordering between ranks. Fenix uses these +ULFM functions internally, so care is taken to ensure that the order of +operations is consistent across ranks. Before any such operation begins, Fenix +first uses MPIX_Comm_agree on the resilient communicator provided by +#Fenix_Init to agree on which 'location' will proceed - if there is any +disagreement, all ranks will enter recovery as if they had detected a failure. +Applications which wish to use these functions themselves should follow this +pattern, providing a unique 'location' value for any operations that may be +interrupted. + +--- + +## Application Recovery + +Once a new communicator has been constructed, application recovery begins. +There are two recovery modes: jumping (default) and non-jumping. With jumping +recovery, Fenix will automatically `longjmp` to the #Fenix_Init call site once +communicator recovery is complete. This allows for very simple recovery logic, +since it mimics the traditional teardown-restart pattern. However, `longjmp` +has many undefined semantics according to the C and C++ specifications and may +result in unexpected behavior due to compiler assumptions and optimizations. +Additionally, some applications may be able to more efficiently recover by +continuing inline. Users can initialize Fenix as non-jumping (see test/no_jump) +to instead return an error code from the triggering MPI function call after +communicator recovery. This may require more intrusive code changes (checking +return statuses of each MPI call). + +Fenix also allows applications to register one or more callback functions with +#Fenix_Callback_register and #Fenix_Callback_pop, which removes the most +recently registered callback. These callbacks are invoked after communicator +recovery, just before control returns to the application. Callbacks are +executed in the reverse order they were registered. + +For C++ applications, it is recommended to use Fenix in non-jumping mode and to +register a callback that throws an exception. At it's simplest, wrapping +everything between #Fenix_Init and #Fenix_Finalize in a single try-catch can +give the same simple recovery logic as jumping mode, but without the undefined +behavior of `longjmp`. + +#Fenix_Init outputs a role, from #Fenix_Rank_role, which helps inform the +application about the recovery state of the rank. It is important to note that +all spare ranks are captured inside #Fenix_Init until they are used for +recovery. Therefore, after recovery, recovered ranks will not have the same +callbacks registered -- recovered ranks will need to manually invoke any +callbacks that use MPI functions. These roles also help the application more +generally modify it's behavior based on each rank's recovery state. diff --git a/include/fenix.h b/include/fenix.h index 77d573b..0a1d783 100644 --- a/include/fenix.h +++ b/include/fenix.h @@ -66,6 +66,18 @@ extern "C" { #include "fenix_data_subset.h" #include "fenix_process_recovery.h" +/** + * @file + * @brief Contains all API function calls and Fenix types. + * This is the only header file a user should include. + */ + +/** + * @defgroup ReturnCodes Return Codes + * @brief All possible return codes from Fenix functions. + * Errors are negative, warnings are positive. + * @{ + */ #define FENIX_SUCCESS 0 #define FENIX_ERROR_UNINITIALIZED -9 #define FENIX_ERROR_NOCATEGORY -10 @@ -91,40 +103,113 @@ extern "C" { #define FENIX_ERROR_CANCELLED -50 #define FENIX_WARNING_SPARE_RANKS_DEPLETED 100 #define FENIX_WARNING_PARTIAL_RESTORE 101 +/**@}*/ -#define FENIX_DATA_GROUP_WORLD_ID 10 -#define FENIX_GROUP_ID_MAX 11 -#define FENIX_TIME_STAMP_MAX 12 -#define FENIX_DATA_MEMBER_ALL 15 -#define FENIX_DATA_MEMBER_ATTRIBUTE_BUFFER 11 -#define FENIX_DATA_MEMBER_ATTRIBUTE_COUNT 12 -#define FENIX_DATA_MEMBER_ATTRIBUTE_DATATYPE 13 -#define FENIX_DATA_MEMBER_ATTRIBUTE_SIZE 14 -#define FENIX_DATA_SNAPSHOT_LATEST -1 -#define FENIX_DATA_SNAPSHOT_ALL 16 -#define FENIX_DATA_SUBSET_CREATED 2 - +//!@internal @brief Agreement code for error handler #define FENIX_ERRHANDLER_LOC 1 +//!@internal @brief Agreement code for finalize #define FENIX_FINALIZE_LOC 2 +//!@internal @brief Agreement code for data commit barrier #define FENIX_DATA_COMMIT_BARRIER_LOC 4 -#define FENIX_DATA_POLICY_IN_MEMORY_RAID 13 +/** + * @defgroup ProcessRecovery Process Recovery + * @brief Functions for managing process recovery in Fenix. + * @details @include{doc} ProcessRecovery.md + * @{ + */ + +/** + * @brief All possible roles returned by Fenix_Init + * + * Describes the current process's state in reference + * to process recovery. + * + * It is important to note that FENIX_ROLE_RECOVERED_RANK + * is only guaranteed to be the value after a single failure, + * so users ought not use the role to directly ensure a valid + * state if they desire to be resilient to failures during their + * failure recovery process. + */ typedef enum { + //!No failures have occurred yet FENIX_ROLE_INITIAL_RANK = 0, + //!This rank was a spare before the most recent failure, or was just spawned FENIX_ROLE_RECOVERED_RANK = 1, + //!This rank was not a spare before the most recent failure FENIX_ROLE_SURVIVOR_RANK = 2 } Fenix_Rank_role; -typedef struct { - MPI_Request mpi_send_req; - MPI_Request mpi_recv_req; -} Fenix_Request; - -extern const Fenix_Data_subset FENIX_DATA_SUBSET_FULL; -extern const Fenix_Data_subset FENIX_DATA_SUBSET_EMPTY; - +/** + * @fn void Fenix_Init(int* role, MPI_Comm comm, MPI_Comm* newcomm, int** argc, char*** argv, int spare_ranks, int spawn, MPI_Info info, int* error); + * @brief Build a resilient communicator and set the restart point. + * + * This function must be called by all ranks in \c comm, after MPI initialization. All calling ranks must + * pass the same values for the parameters \c comm, \c spare_ranks, \c spawn, and \c info. \c Fenix_init + * must be called exactly once by each rank. This function is used (1) to activate the Fenix library, (2) + * to specify extra resources in case of rank failure, and (3) to create a logical resumption point in case + * of rank failure. + * + * For C, the program may rely on the the state of any variables defined and set before the call to \c Fenix_Init. + * But note that the code executed before \c Fenix_Init is executed by all ranks in the system (including spare + * ranks, see below). For C++, the state of objects declared before \c Fenix_Init but within the same scope as + * \c Fenix_Init is compiler-dependant, and it is recommended to place \c Fenix_Init within a subscope exluding + * any variables expected to no be destructed. + * + * It is recommended to access argc and argv only after executin \c Fenix_Init, since command line arguments + * passed to this function that apply to Fenix may be removed by \c Fenix_Init. + * + * \c Fenix_Init is blocking in the following sense. If it is entered for the first time via a regular, explicit + * function call, it must be entered by all ranks in communicator \c comm. If it is entered after an error + * intercepted by Fenix (it if the default execution resumption point, see _info below), no ranks are allowed + * to exit from it until all *non-failed* ranks have returned control to it. **Note**: Typically control is + * returned automatically through revocation of the resilient communicator, which means ranks which have long + * delays between MPI function calls or ranks which only use communicators unaffected by failure may lead to + * long delays between a failure and its recovery. + * + * Ranks to be used as spare ranks by Fenix will be available to the application only before \c Fenix_Init, + * or after they are used to replace a failed rank, in which case they turn into active ranks. This document + * refers to the latter as \c RECOVERED ranks (see #Fenix_Rank_role). Note that all spare + * ranks that have not been used to recover from failures (and, therefore, are still reserved by Fenix and kept + * inside \c Fenix_Init) will automatically call \c MPI_Finalize and exit when all active ranks have entered the + * #Fenix_Finalize call. + * + * No Fenix functions may be called before \c Fenix_Init, except #Fenix_Initialized. + * + * @param[out] role The current role of this rank (see #Fenix_Rank_role) + * @param[in] comm The base communicator to construct a resilient communicator from, + * which must include any spare ranks (see below) the user deems necessary. + * MPI_COMM_WORLDis a valid value, but MPI_COMM_SELF is not. + * @param[out] newcomm Resilient output communicator, managed by Fenix and derived + * from comm, to be used by the application instead of comm. + * @param[inout] argc Pointer to application main's argc parameter + * @param[inout] argv Pointer to application main's argv parameter + * @param[in] spare_ranks The number of ranks in comm that are exempted by Fenix + * in the construction of the resilient communicator by Fenix_Init. These ranks + * are kept in reserve to substitute for failed ranks. Failed ranks in resilient + * communicators are replaced by spare or spawned ranks. + * @param[in] spawn *Unimplemented*: Whether to enable spawning new ranks to replace + * failed ranks when spares are unavailable. + * @param[in] info Fenix recovery configuration parameters, may be MPI_INFO_NULL + * Supports the "FENIX_RESUME_MODE" key, used to indicate where execution should resume upon + * rank failure for all active (non-spare) ranks in any resilient communicators, not only for + * those ranks in communicators that failed. The following values associated with the + * "resume_mode" key are supported: + * - "Fenix_init" (default): execution resumes at logical exit of Fenix_Init. + * - "NO_JUMP": execution continues from the failing MPI call. Errors are otherwise handled + * as normal, but return the error code as well. Applications should typically + * either check for return codes or assign an error callback through Fenix. + * @param[out] error The return status of \c Fenix_Init
+ * Used to signal that a non-fatal error or special condition was encountered in the execution of + * Fenix_Init, or FENIX_SUCCESS otherwise. It has the same value across all ranks released by + * Fenix_Init. If spawning is explicitly disabled (_spawn equals false) and spare ranks have been + * depleted, Fenix will repair resilience communicators by shrinking them and will report such + * shrinkage in this return parameter through the value FENIX_WARNING_SPARE_RANKS_DEPLETED. + */ + +//!@internal #define Fenix_Init(_role, _comm, _newcomm, _argc, _argv, _spare_ranks, \ _spawn, _info, _error) \ { \ @@ -138,100 +223,475 @@ extern const Fenix_Data_subset FENIX_DATA_SUBSET_EMPTY; __fenix_postinit( _error ); \ } -int Fenix_Initialized(int *); +/** + * @brief Sets flag to true if Fenix_Init has been called, else false. + * @param[out] flag Pointer to the flag to be set. + * @returnstatus + */ +int Fenix_Initialized(int *flag); + +/** + * @brief Register a callback to be invoked after failure process recovery. + * + * This function registers a callback to be invoked after a failure has been recovered by Fenix, + * and right before resuming application execution (e.g. returning from #Fenix_Init by default). + * If this function is called more than once, the different callbacks will be called in the + * reverse order that they were registered (i.e. as a callback stack). + * + * Callback functions are passed the newly-repaired resilient communicator, the error code returned + * by MPI in the communication action which caused a failure recovery, and the user-provided \c void* + * callback data. + * + * Callbacks will only be invoked by survivor ranks, since spare ranks or respawned ranks had no way + * to register them before a failure. + * + * @param[in] recover the callback function to register. + * @param[in] callback_data The user-provided data which will be passed to the callback. + * + * @returnstatus + */ int Fenix_Callback_register(void (*recover)(MPI_Comm, int, void *), void *callback_data); +/** + * @brief Pop the most recently registered callback from the callback stack. + * @returnstatus + */ int Fenix_Callback_pop(); +/** + * @brief Check for any failed ranks + * + * @param[in] do_recovery If true, Fenix will attempt to recover from any detected failures. + * Else, it will ignore any failures and simply return the MPI return code. + * @return MPI_SUCCESS if no failures were detected, else the MPI return code. + */ +int Fenix_Process_detect_failures(int do_recovery); + +//!@unimplemented Returns the number of ranks with a given #Fenix_Rank_role int Fenix_get_number_of_ranks_with_role(int, int *); +//!@unimplemented Returns the #Fenix_Rank_role for a given rank int Fenix_get_role(MPI_Comm comm, int rank, int *role); +/** + * @brief Get the list of ranks that failed in the most recent failure. + * @param[out] fail_list Set to a list of failed ranks. + * @return The number of failed ranks. + */ +int Fenix_Process_fail_list(int** fail_list); + +/** + * @brief Check a pre-recovery request without error + * @param[in] request The request to check + * @param[out] status The status of the request + * @return True if the request was cancelled or has unknown completion status, + * false if it completed successfully. + */ +int Fenix_check_cancelled(MPI_Request *request, MPI_Status *status); + + +/** + * @brief Clean up Fenix state. Each active rank must call \c Fenix_Finalize before exiting. + * + * This function cleans up all Fenix state, if any. If an MPI program using the Fenix library terminates + * normally (i.e. not due to a call to \c MPI_Abort, or an unrecoverable error) then each rank must call + * \c Fenix_Finalize before it exits. It must be called before \c MPI_Finalize, and after #Fenix_Init. + * There shall be no function calls after this function, except #Fenix_Initialized. + * + * As noted in the description of #Fenix_Init, all spare ranks that have not been used to + * recover from failures (and therefore are still reserved by Fenix and kept inside #Fenix_Init) will call + * \c MPI_Finalize and exit when all active ranks have called \c Fenix_Finalize. + * + * **Advice**: Sometimes users may want to remove ranks proactively from the execution, for example because + * monitoring data shows that failure of a rank is imminent or that a rank is executing un-manageably slowly. + * This can be accomplished by calling \c exit on the targeted ranks, followed by an invocation of MPI_Barrier. + * The removed ranks will be reported as failed and error handling will progress appropriately. No calls to finalize + * are needed in this case. + */ int Fenix_Finalize(); -int Fenix_Data_group_create(int group_id, MPI_Comm, int start_time_stamp, +/**@}*/ + + +/** + * @defgroup DataRecovery Data Recovery + * @brief Functions for storing and restoring data in Fenix. + * @details @include{doc} DataRecovery.md + * + * @{ + */ +#define FENIX_DATA_GROUP_WORLD_ID 10 +#define FENIX_GROUP_ID_MAX 11 +#define FENIX_TIME_STAMP_MAX 12 +#define FENIX_DATA_MEMBER_ALL 15 +#define FENIX_DATA_MEMBER_ATTRIBUTE_BUFFER 11 +#define FENIX_DATA_MEMBER_ATTRIBUTE_COUNT 12 +#define FENIX_DATA_MEMBER_ATTRIBUTE_DATATYPE 13 +#define FENIX_DATA_MEMBER_ATTRIBUTE_SIZE 14 +#define FENIX_DATA_SNAPSHOT_LATEST -1 +#define FENIX_DATA_SNAPSHOT_ALL 16 +#define FENIX_DATA_SUBSET_CREATED 2 + +#define FENIX_DATA_POLICY_IN_MEMORY_RAID 13 + +/** + * @unimplemented As MPI_Request, but for Fenix asynchronous data recovery calls + */ +typedef struct { + MPI_Request mpi_send_req; + MPI_Request mpi_recv_req; +} Fenix_Request; + +//!@brief A standin for checkpointing/recovering all available data in a member. +extern const Fenix_Data_subset FENIX_DATA_SUBSET_FULL; + +//!@brief A standin for checkpointing/recovering none of the available data in a member. +extern const Fenix_Data_subset FENIX_DATA_SUBSET_EMPTY; + + +/** + * @brief Create a Data Group + * @qualifier collective + * + * If a group with this group_id was already created in the past and has not been deleted, the + * parameters of this call are ignored and this function simply serves to coordinate with any + * ranks that have not yet created this group (e.g. due to a failure). + * + * All calling ranks must pass the same values for the parameters \c group_id, \c comm, + * \c start_time_stamp, \c policy_name, and \c policy_value. + * + * @param group_id A unique identifier to this group. + * @param comm A resilient communicator on which the group is formed. + * @param start_time_stamp The time_stamp to be used for the first commit in this group. + * @param depth + * @parblock + * The number of successive snapshots of this group that are retained by Fenix, in + * addition to the most recent one, and that can be recovered by calling Fenix data member + * restore functions. + * + * For example, a depth of 0 means Fenix will keep only the necessary data to restore the + * most recent snapshot, freeing or overwriting older snapshots automatically. A depth + * of -1 is currently not supported, but would ordinarily indicate that no snapshots should + * be removed automatically. + * @endparblock + * @param policy_name Currently, may only be FENIX_DATA_POLICY_IN_MEMORY_RAID + * @param policy_value Pointer to data passed along to the policy. + * See the specific policy for more information. + * @param flag pointer to store policy-specific status or errors + * @return FENIX_SUCCESS, or an error value. + */ +int Fenix_Data_group_create(int group_id, MPI_Comm comm, int start_time_stamp, int depth, int policy_name, void* policy_value, int* flag); +/** + * @brief Create a data member for store/restore operations + * @qualifier collective + * @qualifier local + * + * All calling ranks in the group's communicator must pass the same values for the parameters + * \c member_id, \c datatype, and \c group_id. + * + * @param group_id Identifier to a data group within which to create the member. + * @param member_id An integer unique within the data group that identifies the data in + * \c source_buffer. Must be nonnegative and less than FENIX_MEMBER_ID_MAX, which is + * guaranteed to be at least 2^30. + * @param buffer Address of the data to be copied to redundant storage maintained by Fenix. + * Note that this parameter may also be specified using #Fenix_Data_member_attr_set, which + * is critical for non-survivor ranks after a failure which will have an invalid address + * which was generated on the failed rank and must update. + * @param count The maximum number of contiguous elements of type \c datatype of the data to be + * stored. Need not be the same in all calling ranks. + * @param datatype The MPI_Datatype of the elements in \c source_buffer + * + * @return FENIX_SUCCESS, or an error value. + */ int Fenix_Data_member_create(int group_id, int member_id, void *buffer, int count, MPI_Datatype datatype); +/** + * @brief Get the storage policy of a data group + * + * @param group_id Identified to the data group to query + * @param policy_name The identifier of the policy name of the data group. + * @param policy_value A location within which to store the policy_values this group's + * policy was configured with. + * @param flag A location set to true if a policy value was extracted, else false. + * @return FENIX_SUCCESS, or an error value. + */ int Fenix_Data_group_get_redundancy_policy(int group_id, int* policy_name, void *policy_value, int *flag); +//!@unimplemented Block on completion of the store operation specified by the request. int Fenix_Data_wait(Fenix_Request request); + +//!@unimplemented Query completion of the store operation specified by the request. int Fenix_Data_test(Fenix_Request request, int *flag); + +/** + * @brief Store a particular group member into the group's resilient storage space, in uncommitted storage. + * @qualifier collective + * + * The user can safely modify the member's data buffer after this call, as the current state is copied immediately. + * Multiple calls may be used to incrementally store data (using subset_specifiers), or overwrite old data prior to a commit. + * + * @param group_id All ranks must provide the same group_id + * @param member_id All ranks must provide the same member_id + * @param subset_specifier Which subset of the data to store. It is always valid for every rank to provide the same + * subset_specifier; depending on the group's policy, varying combinations of specifiers may be possible. + * @return FENIX_SUCCESS, or an error value. + */ int Fenix_Data_member_store(int group_id, int member_id, Fenix_Data_subset subset_specifier); -int Fenix_Data_member_storev(int member_id, int group_id, + +//!@unimplemented As [store](#Fenix_Data_member_store), but subsets may vary rank-to-rank. +int Fenix_Data_member_storev(int group_id, int member_id, Fenix_Data_subset subset_specifier); -int Fenix_Data_member_istore(int member_id, int group_id, +//!@unimplemented As [store](#Fenix_Data_member_store), but asynchronous. +int Fenix_Data_member_istore(int group_id, int member_id, Fenix_Data_subset subset_specifier, Fenix_Request *request); -int Fenix_Data_member_istorev(int member_id, int group_id, +//!@unimplemented As [istore](#Fenix_Data_member_istore), but asynchronous. +int Fenix_Data_member_istorev(int group_id, int member_id, Fenix_Data_subset subset_specifier, Fenix_Request *request); +/** + * @brief Commit stored data members to the group's next snapshot. + * @qualifier collective + * @qualifier local + * + * This function is used to freeze the current state of a data group, + * together with all its application data that has been stored in Fenix’ + * redundant storage, and label it with a time stamp, thus creating a + * snapshot of the stored application data. Only data that has been + * committed is eligible for recovery through #Fenix_Data_member_restore. + * An application needs to call #Fenix_Data_wait for all pending asynchronous + * [Fenix_Data_member_istore(v)](@ref Fenix_Data_member_istore) operations + * in the group before committing. + * + * @param[in] group_id The group to commit + * @param[out] time_stamp The time stamp of the new snapshot + * @returnstatus + */ int Fenix_Data_commit(int group_id, int *time_stamp); +/** + * @brief As [commit](#Fenix_Data_commit), but ensures a globally consistent commit. + * @qualifier collective + * + * This function does not function as a traditional barrier. + * The commit will proceed if all *non-failed* ranks reach the barrier. + * This allows for commits to be made when a rank fails after storing all + * of its data into resilient storage. + * + * @param[in] group_id The group to commit + * @param[out] time_stamp The time stamp of the new snapshot + * @returnstatus + */ int Fenix_Data_commit_barrier(int group_id, int *time_stamp); +//!@unimplemented Block until all ranks in the group have reached this point. int Fenix_Data_barrier(int group_id); +/** + * @brief Restore the data of a group member from a snapshot. + * @qualifier collective + * + * All ranks in the group’s resilient communicator must pass the + * same values for the parameters group_id, member_id, and time_stamp. + * This function is used to retrieve data from consistent snapshot + * members. This function can only be used if the size of the + * communicator used to store the data is the same as that at the time + * of data recovery (this implies non-shrinking communicator recovery + * in case of a rank loss). + * + * If the size of the buffer needing to receive the recovery data is + * unknown for a particular rank, it can be queried using + * #Fenix_Data_member_attr_get. + * + * @param[in] group_id The group to restore from + * @param[in] member_id The member to restore + * @param[out] target_buffer The buffer to store the restored data + * @param[in] max_count The maximum number of elements to restore + * @param[in] time_stamp The time stamp of the snapshot to restore from + * @param[out] found_data The subset of the data that was found in the snapshot + * @returnstatus + */ int Fenix_Data_member_restore(int group_id, int member_id, void *target_buffer, int max_count, int time_stamp, Fenix_Data_subset* found_data); +/** + * @brief Local-only version of Fenix_Data_member_restore + * + * This function restores the data of a group member from the local + * snapshot. + * + * @param[in] group_id The group to restore from + * @param[in] member_id The member to restore + * @param[out] target_buffer The buffer to store the restored data + * @param[in] max_count The maximum number of elements to restore + * @param[in] time_stamp The time stamp of the snapshot to restore from + * @param[out] found_data The subset of the data that was found in the snapshot + * @returnstatus + */ int Fenix_Data_member_lrestore(int group_id, int member_id, void *target_buffer, int max_count, int time_stamp, Fenix_Data_subset* found_data); +//!@unimplemented As #Fenix_Data_member_restore, but restores from a specific rank's data. int Fenix_Data_member_restore_from_rank(int member_id, void *data, int max_count, int time_stamp, int group_id, int source_rank); +/** + * @brief Create a data subset for use in store operations. + * + * Creates a subset based on num_blocks pairs of + * {start_offset,end_offset}, + * {start_offset+stride,end_offset+stride}, + * {start_offset+2*stride,end_offset+2*stride}, + * etc. + * + * The value of start_offset must be smaller than or equal + * to the value of end_offset to indicate non-negative block + * size. Otherwise, the function returns an error code. + * + * Created subsets must be deleted with #Fenix_Data_subset_delete + * to free memory. + * + * @param[in] num_blocks The number of contiguous data blocks. + * @param[in] start_offset The index of the first element in the first data block. + * @param[in] end_offset The index of the last element in the first data block. + * @param[in] stride Regular shift between successive data blocks. + * @param[out] subset_specifier The created subset. + * @returnstatus + */ int Fenix_Data_subset_create(int num_blocks, int start_offset, int end_offset, int stride, Fenix_Data_subset *subset_specifier); +/** + * @brief As #Fenix_Data_subset_create, but with varying start and end offsets. + * + * Creates a subset based on num_blocks pairs of {start_offset,end_offset}. + * The value of start_offset must be smaller than or equal to end_offset + * to indicate non-negative block size. Otherwise, the function returns an + * error code. + * + * Created subsets must be deleted with #Fenix_Data_subset_delete + * to free memory. + * + * @param[in] num_blocks The number of contiguous data blocks. + * @param[in] array_start_offsets The index of the first element in each data block. + * @param[in] array_end_offsets The index of the last element in each data block. + * @param[out] subset_specifier The created subset. + */ int Fenix_Data_subset_createv(int num_blocks, int *array_start_offsets, int *array_end_offsets, Fenix_Data_subset *subset_specifier); +/** + * @brief Delete a data subset. + * + * Frees the memory associated with a data subset object. + * + * @param[in] subset_specifier The subset to delete. + * @returnstatus + */ int Fenix_Data_subset_delete(Fenix_Data_subset *subset_specifier); +//!@unimplemented Get the number of members in a data group. int Fenix_Data_group_get_number_of_members(int group_id, int *number_of_members); -int Fenix_Data_group_get_member_at_position(int position, int *member_id, - int group_id); - +//!@unimplemented Get member ID based on member index +int Fenix_Data_group_get_member_at_position(int group_id, int *member_id, + int position); + +/** + * @brief Get the number of locally-available snapshots in a data group. + * + * May include snapshots that are inconsistent across the group. + * + * @param[in] group_id The group to query + * @param[out] number_of_snapshots The number of snapshots in the group + * @returnstatus + */ int Fenix_Data_group_get_number_of_snapshots(int group_id, int *number_of_snapshots); +/** + * @brief Get the time stamp of a snapshot at a given index. + * + * Snapshots are indexed in reverse order in which the user committed them + * (e.g. the most recent available snapshot has position=0). + * + * @param[in] group_id The group to query + * @param[in] position The index of the snapshot, which must be [0, number_of_snapshots) + * @param[out] time_stamp The time stamp of the snapshot + * + */ int Fenix_Data_group_get_snapshot_at_position(int group_id, int position, int *time_stamp); +//!@unimplemented Get the value of a member's attribute. int Fenix_Data_member_attr_get(int group_id, int member_id, int attributename, void *attributevalue, int *flag, int source_rank); +/** + * @brief Set the value of a member's attribute. + * + * Valid names are #FENIX_DATA_MEMBER_ATTRIBUTE_BUFFER, #FENIX_DATA_MEMBER_ATTRIBUTE_COUNT, + * and #FENIX_DATA_MEMBER_ATTRIBUTE_DATATYPE. + * + * The COUNT and DATATYPE attributes may only be set before the first store operation. + * Contrary to the Fenix specification, returning to #Fenix_Init after a failure does not + * allow the user to set these attributes again. + * + * @param[in] group_id The group to update + * @param[in] member_id The member to update + * @param[in] attribute_name The attribute to update + * @param[in] attribute_value The new value of the attribute + * @param[out] flag Set to true if the attribute was set, else false + * @returnstatus + */ int Fenix_Data_member_attr_set(int group_id, int member_id, int attribute_name, void *attribute_value, int *flag); +/** + * @brief Delete a snapshot from a data group. + * @qualifier local + * + * @param[in] group_id The group to delete from + * @param[in] time_stamp The time stamp of the snapshot to delete + * @returnstatus + */ int Fenix_Data_snapshot_delete(int group_id, int time_stamp); +/** + * @brief Delete a data group. + * @qualifier local + * + * @param[in] group_id The group to delete + * @returnstatus + */ int Fenix_Data_group_delete(int group_id); +/** + * @brief Delete a data member. + * @qualifier local + * + * @param[in] group_id The group to delete from + * @param[in] member_id The member to delete + * @returnstatus + */ int Fenix_Data_member_delete(int group_id, int member_id); - -int Fenix_Process_fail_list(int** fail_list); - -int Fenix_check_cancelled(MPI_Request *request, MPI_Status *status); - -int Fenix_Process_detect_failures(int do_recovery); +/**@}*/ #if defined(c_plusplus) || defined(__cplusplus) } From c1d8dd53b607765a9eeeabc058db50db93c5110e Mon Sep 17 00:00:00 2001 From: Matthew Whitlock Date: Fri, 22 Nov 2024 12:57:40 -0500 Subject: [PATCH 13/13] Remove doxygen branch docs, logging fixes --- .github/scripts/build-gh-pages.sh | 14 ++++++++++---- .github/workflows/docs.yml | 1 - doc/html/CMakeLists.txt | 4 ++-- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/.github/scripts/build-gh-pages.sh b/.github/scripts/build-gh-pages.sh index 894d71d..9143021 100644 --- a/.github/scripts/build-gh-pages.sh +++ b/.github/scripts/build-gh-pages.sh @@ -3,18 +3,24 @@ set -e +#Run with sudo if not root user +SUDO="" +if [ $(id -u) -ne 0 ]; then + SUDO="sudo" +fi + echo "Installing apt packages" -sudo apt-get update >/dev/null -sudo apt-get install -y wget git cmake graphviz >/dev/null +$SUDO apt-get update >/dev/null +$SUDO apt-get install -y wget git cmake graphviz >/dev/null echo "Installing Doxygen" -wget https://www.doxygen.nl/files/doxygen-1.12.0.linux.bin.tar.gz >/dev/null +wget -q https://www.doxygen.nl/files/doxygen-1.12.0.linux.bin.tar.gz tar -xzf doxygen-1.12.0.linux.bin.tar.gz >/dev/null export PATH="$PWD/doxygen-1.12.0/bin:$PATH" #List of branches to build docs for #TODO: Remove doxygen branch once tested -BRANCHES="doxygen master develop" +BRANCHES="master develop" build-docs() ( git checkout $1 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 1f6609f..4a3acba 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -5,7 +5,6 @@ on: branches: - master - develop - - doxygen # TODO: Remove after testing #Only one of this workflow runs at a time concurrency: diff --git a/doc/html/CMakeLists.txt b/doc/html/CMakeLists.txt index 70677f8..62a82ab 100644 --- a/doc/html/CMakeLists.txt +++ b/doc/html/CMakeLists.txt @@ -26,8 +26,6 @@ if("html" IN_LIST DOC_VERSIONS) list(REMOVE_ITEM DOC_VERSIONS "html") endif() -message(STATUS "Existing documentation versions: ${FENIX_DOC_VERSIONS}") - list(APPEND DOC_VERSIONS ${DOXYGEN_HTML_OUTPUT}) list(REMOVE_DUPLICATES DOC_VERSIONS) list(SORT DOC_VERSIONS) @@ -36,6 +34,8 @@ if("main" IN_LIST DOC_VERSIONS) list(PREPEND DOC_VERSIONS "main") endif() +message(STATUS "Documentation versions: ${DOC_VERSIONS}") + set(DOC_DEFAULT_VERSION "develop") if(NOT DOC_DEFAULT_VERSION IN_LIST DOC_VERSIONS) set(DOC_DEFAULT_VERSION ${FENIX_BRANCH})