From 5d4fd6a2ccf37d95b8a5917db7be95f5e830ca5a Mon Sep 17 00:00:00 2001
From: "Jian Zhang (James)" <6593865@qq.com>
Date: Mon, 9 Oct 2023 23:45:35 -0700
Subject: [PATCH 1/6] [Doc] Add return_proba argument explanation (#536)

*Issue #, if available:*

*Description of changes:*
This PR fix the documentation of `return_proba`:
- Add the return_proba argument;
- Add a note on eval_metric.

By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice.

---------

Co-authored-by: Ubuntu <ubuntu@ip-172-31-35-48.us-west-2.compute.internal>
---
 docs/source/configuration/configuration-run.rst | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/docs/source/configuration/configuration-run.rst b/docs/source/configuration/configuration-run.rst
index 428d35f19e..8866e348a9 100644
--- a/docs/source/configuration/configuration-run.rst
+++ b/docs/source/configuration/configuration-run.rst
@@ -26,7 +26,7 @@ GraphStorm's `graphstorm.run.launch <https://github.com/awslabs/graphstorm/blob/
     - NCCL_DEBUG=INFO
 - **lm-encoder-only**: Indicate that the model is using language model + decoder only. model. No GNN is involved, only graph structure.
 
-..  note:: Below configurations can be set either in a YAML configuraiton file or be added as arguments of launch command.
+..  note:: Below configurations can be set either in a YAML configuration file or be added as arguments of launch command.
 
 Environment Configurations
 -------------------------------------
@@ -275,7 +275,7 @@ General Configurations
     - Yaml: ``task_type: node_classification``
     - Argument: ``--task-type node_classification``
     - Default value: This parameter must be provided by user.
-- **eval_metric**: Evaluation metric used during evaluation. The input can be a string specifying the evaluation metric to report or a list of strings specifying a list of evaluation metrics to report. The first evaluation metric is treated as the major metric and is used to choose the best trained model. The supported evaluation metrics of classification tasks include ``accuracy``, ``precision_recall``, ``roc_auc``, ``f1_score``, ``per_class_f1_score``. The supported evaluation metrics of regression tasks include ``rmse``, ``mse`` and ``mae``. The supported evaluation metrics of link prediction tasks include ``mrr``.
+- **eval_metric**: Evaluation metric used during evaluation. The input can be a string specifying the evaluation metric to report or a list of strings specifying a list of evaluation metrics to report. The first evaluation metric is treated as the major metric and is used to choose the best trained model. The supported evaluation metrics of classification tasks include ``accuracy``, ``precision_recall``, ``roc_auc``, ``f1_score``, ``per_class_f1_score``. The supported evaluation metrics of regression tasks include ``rmse``, ``mse`` and ``mae``. The supported evaluation metrics of link prediction tasks include ``mrr``. 
     - Yaml: ``eval_metric:``
         | ``- accuracy``
         | ``- precision_recall``
@@ -307,6 +307,10 @@ Classification and Regression Task
     - Yaml: ``imbalance_class_weights: 0.1,0.2,0.3``
     - Argument: ``--imbalance-class-weights 0.1,0.2,0.3``
     - Default value: ``None``
+- **return_proba**: For classification task, this configuration determines whether to return probability estimates for each class or the maximum probable class. Set `true`` to return probability estimates and `false` to return the maximum probable class.
+    - Yaml: ``return_proba: true``
+    - Argument: ``--return_proba true``
+    - Default value: ``true``
 - **save_prediction_path**: Path to save prediction results. This is used in node/edge classification/regression inference.
     - Yaml: ``save_prediction_path: /data/infer-output/predictions/``
     - Argument: ``--save-prediction-path /data/infer-output/predictions/``

From 1a912c311e07301534ce99ed3f5c978c2d534151 Mon Sep 17 00:00:00 2001
From: "Jian Zhang (James)" <6593865@qq.com>
Date: Tue, 10 Oct 2023 12:19:23 -0700
Subject: [PATCH 2/6] [Doc] Fix bolding font format error in the
 configuration-run rst file (#539)

*Issue #, if available:*

*Description of changes:*
The previous configuration of training and inference has the bolding
font issue. This PR fix this issue by adding a space line after each
argument description.

By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice.

Co-authored-by: Ubuntu <ubuntu@ip-172-31-35-48.us-west-2.compute.internal>
---
 .../configuration/configuration-run.rst       | 82 ++++++++++++++++++-
 1 file changed, 80 insertions(+), 2 deletions(-)

diff --git a/docs/source/configuration/configuration-run.rst b/docs/source/configuration/configuration-run.rst
index 8866e348a9..7ff2372bd4 100644
--- a/docs/source/configuration/configuration-run.rst
+++ b/docs/source/configuration/configuration-run.rst
@@ -31,10 +31,12 @@ GraphStorm's `graphstorm.run.launch <https://github.com/awslabs/graphstorm/blob/
 Environment Configurations
 -------------------------------------
 - **backend**: (**Required**) PyTorch distributed backend, the suggested backend is gloo. Support backends include gloo and nccl
+
     - Yaml: ``backend: gloo``
     - Argument: ``--backend gloo``
     - Default value: ``gloo``
 - **verbose**: Set true to print more execution information
+
     - Yaml: ``verbose: false``
     - Argument: ``--verbose false``
     - Default value: ``false``
@@ -44,24 +46,29 @@ Model Configurations
 GraphStorm provides a set of parameters to config the GNN model structure (input layer, gnn layer, decoder layer, etc)
 
 - **model_encoder_type**: (**Required**) The Encoder module used to encode graph data. It can be a GNN encoder or a non-GNN encoder. A GNN encoder is composed of an input module, which encodes input node features, and a GNN module. A non-GNN encoder only contains an input module. GraphStorm supports two GNN encoders: `rgcn` which uses relational graph convolutional network as its GNN module and `rgat` which uses relational graph attention network as its GNN module. GraphStorm supports two non-GNN encoder: `lm` which requires each node type has and only has text features and uses language model, e.g., Bert, to encode these features and `mlp` which accepts various types of input node features (text feature, floating points and learnable embeddings) and finally uses an MLP to project these features into same dimension.
+
     - Yaml: ``model_encoder_type: rgcn``
     - Argument: ``--model-encoder-type rgcn``
     - Default value: This parameter must be provided by user.
 - **node_feat_name**: User defined feature name. It accepts two formats: a) `fname`, if a node has node features, the corresponding feature name will be fname; b) `ntype0:feat0 ntype1:featA ...`, different node types have different node feature name(s). In the example, "ntype0" has a node feature named "feat0" and "ntype1" has a node feature named "featA". Note: Characters `:` and ` ` are not allowed to be used in node feature names.  And in Yaml format, need to put each node's feature in a separated line that starts with a hyphon.
+
     - Yaml: ``node_feat_name:``
                 | ``- "ntype1:featA"``
                 | ``- "ntype0:feat0"``
     - Argument: ``--node-feat-name "ntype0:feat0 ntype1:featA"``
     - Default value: If not provided, there will be no node features used by GraphStorm even graphs have node features attached.
 - **num_layers**: Number of GNN layers. Must be an integer larger than 0 if given. By default, it is set to 0, which means no GNN layers.
+
     - Yaml: ``num_layers: 2``
     - Argument: ``--num-layers 2``
     - Default value: ``0``
 - **hidden_size**: (**Required**) The dimension of hidden GNN layers. Must be an integer larger than 0. Currently, each GNN layer has the same hidden dimension.
+
     - Yaml: ``hidden_size: 128``
     - Argument: ``--hidden-size 128``
     - Default value: This parameter must be provided by user.
 - **use_self_loop**: Set true include self feature as a special relation in relational GNN models. Used by built-in RGCN and RGAT model.
+
     - Yaml: ``use_self_loop: false``
     - Argument: ``--use-self-loop false``
     - Default value: ``true``
@@ -71,6 +78,7 @@ Built-in Model Specific Configurations
 RGCN
 `````
 - **num_bases**: Number of filter weight matrices. num_bases is used to reduce the overall parameters of a RGCN model. It allows weight metrics of different relation types to share parameters. Note: the number of relation types of the graph used in training must be divisible by num_bases. By default, num_bases is set to -1, which means weight metrics do not share parameters.
+
     - Yaml: ``num_bases: 2``
     - Argument: ``--num-bases 2``
     - Default value: ``-1``
@@ -78,6 +86,7 @@ RGCN
 RGAT
 `````
 - **num_heads**: Number of attention heads.
+
     - Yaml: ``num_heads: 8``
     - Argument: ``--num-heads 8``
     - Default value: ``4``
@@ -87,42 +96,52 @@ Model Save/Restore Configurations
 GraphStorm provides a set of parameters to control how and where to save and restore models.
 
 - **save_model_path**: A path to save GraphStorm model parameters and the corresponding optimizer status. The saved model parameters can be used in inference or model fine-tuning. See restore_model_path for how to retrieve a saved model and restore_optimizer_path for how to retrieve optimizer status.
+
     - Yaml: ``save_model_path: /model/checkpoint/``
     - Argument: ``--save-model-path /model/checkpoint/``
     - Default value: If not provide, models will not be saved.
 - **save_embed_path**: A path to save generated node embeddings.
+
     - Yaml: ``save_embed_path: /model/emb/``
     - Argument: ``--save-embed-path /model/emb/``
     - Default value: If not provide, models will not be saved.
 - **save_model_frequency**: Number of iterations to save model once. By default, GraphStorm will save models at the end of each epoch if save_model_path is provided. A user can set a positive integer, e.g. `N`, to let GraphStorm save models every `N`` iterations (mini-batches).
+
     - Yaml: ``save_model_frequency: 1000``
     - Argument: ``--save-model-frequency 1000``
     - Default value: ``-1``. GraphStorm will not save models within an epoch.
 - **topk_model_to_save**: The number of top best GraphStorm model to save. By default, GraphStorm will keep all the saved models in disk, which will consume huge number of disk space. Users can set a positive integer, e.g. `K`, to let GraphStorm only save `K`` models with the best performance.
+
     - Yaml: ``topk_model_to_save: 3``
     - Argument: ``--topk-model-to-save 3``
     - Default value: ``0``. GraphStorm will save all the saved models in disk.
 - **save_perf_results_path**: Folder path to save performance results of model evaluation.
+
     - Yaml: ``save_perf_results_path: /model/results/``
     - Argument: ``--save-perf-results-path /model/results/``
     - Default value: ``None``
 - **task_tracker**: A task tracker used to formalize and report model performance metrics. Now GraphStorm only supports sagemaker_task_tracker which prints evaluation metrics in a formatted way so that a user can capture those metrics through SageMaker. See Monitor and Analyze Training Jobs Using Amazon CloudWatch Metrics  for more details.
+
     - Yaml: ``task_tracker: sagemaker_task_tracker``
     - Argument: ``--task_tracker sagemaker_task_tracker``
     - Default value: ``sagemaker_task_tracker``
 - **log_report_frequency**: The frequency of reporting model performance metrics through task_tracker. The frequency is defined by using number of iterations, i.e., every N iterations the evaluation metrics will be reported. (Please note the evaluation metrics should be generated at the reporting iteration. See "eval_frequency" for how evaluation frequency is controlled.)
+
     - Yaml: ``log_report_frequency: 1000``
     - Argument: ``--log-report-frequency 1000``
     - Default value: ``1000``
 - **restore_model_path**: A path where GraphStorm model parameters were saved. For training, if restore_model_path is set, GraphStom will retrieve the model parameters from restore_model_path instead of initializing the parameters. For inference, restore_model_path must be provided.
+
     - Yaml: ``restore_model_path: /model/checkpoint/``
     - Argument: ``--restore-model-path /model/checkpoint/``
     - Default value: This parameter must be provided if users want to restore a saved model.
 - **restore_model_layers**: Specify which GraphStorm neural network layers to load. This argument is useful when a user wants to pre-train a GraphStorm model using link prediction and fine-tune the same model on a node or edge classification/regression task. Currently, three neural network layers are supported, i.e., ``embed`` (input layer), ``gnn`` and ``decoder``. A user can select one or more layers to load.
+
     - Yaml: ``restore_model_path: embed``
     - Argument: ``--restore-model-layers embed,gnn``
     - Default value: Load all neural network layers
 - **restore_optimizer_path**: A path storing optimizer status corresponding to GraphML model parameters. This is used when a user wants to fine-tune a model from a pre-trained one.
+
     - Yaml: ``restore_optimizer_path: /model/checkpoint/optimizer``
     - Argument: ``--restore-optimizer-path /model/checkpoint/optimizer``
     - Default value: This parameter must be provided if users want to restore a saved optimizer.
@@ -132,66 +151,82 @@ Model Training Hyper-parameters Configurations
 GraphStorm provides a set of parameters to control training hyper-parameters.
 
 - **fanout**: The fanout of each GNN layers. The fanouts must be integers larger than 0. The number of fanouts must equal to num_layers. It accepts two formats: a) `"20,10"`, it defines number of neighbors to sample per edge type for each GNN layer with the ith element being the fanout for the ith GNN layer. In the example, the fanout of the 0th GNN layer is 20 and the fanout of the 1st GNN layer is 10. b) `\"etype2:20\@etype3:20\@etype1:10,etype2:10\@etype3:4\@etype1:2\"`. It defines the numbers of neighbors to sample for different edge types for each GNN layers with the i-th element being the fanout for the i-th GNN layer. In the example, the fanouts of etype2, etype3 and etype1 of 0th GNN layer are 20, 20 and 10 respectively and the fanouts of etype2, etype3 and etype1 of 0th GNN layer are 10, 4 and 2 respectively. Each etype (e.g., etype2) should be a canonical etype in format of `\"srcntype/relation/dstntype\"`
+
     - Yaml: ``fanout: 10,10``
     - Argument: ``--fanout 10,10``
     - Default value: This parameter must be provided by user. But if set the ``--num_layers`` to be 0, which means there is no GNN layer, no need to specify this configuration.
 - **dropout**: Dropout probability. Dropout must be a float value in [0,1). Dropout is applied to every GNN layer(s).
+
     - Yaml: ``dropout: 0.5``
     - Argument: ``--dropout 0.5``
     - Default value: ``0.0``
 - **lr**: (**Required**) Learning rate. Learning rate for dense parameters of input encoder, model encoder and decoder.
+
     - Yaml: ``lr: 0.5``
     - Argument: ``--lr 0.5``
     - Default value: This parameter must be provided by user.
 - **max_grad_norm**: Gradient clip which limits the magnitude of gradients during training in order to prevent issues like exploding gradients and improve the stability and convergence of the training process.
+
     - Yaml: ``max_grad_norm: 0.1``
     - Argument: ``--max-grad-norm 0.1``
     - Default value: None
-- **grad_norm_type**: Type of norm that is used to compute the gradient norm
+- **grad_norm_type**: Type of norm that is used to compute the gradient norm.
+
     - Yaml: ``grad_norm_type: inf``
     - Argument: ``grad_norm_type 2``
     - Default value: 2.0
 - **num_epochs**: Number of training epochs. Must be integer.
+
     - Yaml: ``num_epochs: 5``
     - Argument: ``--num-epochs 5``
     - Default value: ``0``. By default only do testing/inference.
 - **batch_size**: (**Required**) Mini-batch size. It defines the batch size of each trainer. The global batch size equals to the number of trainers multiply the batch_size. For example, suppose we have 2 machines each with 8 GPUs and set batch_size to 128. The global batch size will be 2 * 8 * 128 = 2048.
+
     - Yaml: ``batch_size: 128``
     - Argument: ``--batch_size 128``
     - Default value: This parameter must be provided by user.
 - **sparse_optimizer_lr**: Learning rate of sparse optimizer. Learning rate for the optimizer corresponding to learnable sparse embeddings.
+
     - Yaml: ``sparse_optimizer_lr: 0.5``
     - Argument: ``--sparse-optimizer-lr 0.5``
     - Default value: Same as lr.
 - **use_node_embeddings**: Set true to use extra learnable node embedding for each node.
+
     - Yaml: ``use_node_embeddings: true``
     - Argument: ``--use-node-embeddings true``
     - Default value: ``false``
 - **wd_l2norm**: Weight decay used by torch.optim.Adam.
+
     - Yaml: ``wd_l2norm: 0.1``
     - Argument: ``--wd-l2norm 0.1``
     - Default value: ``0``
 - **alpha_l2norm**: Coefficiency of the l2 norm of dense parameters. GraphStorm adds a regularization loss, i.e., l2 norm of dense parameters, to the final loss. It uses alpha_l2norm to re-scale the regularization loss. Specifically, loss = loss +  alpha_l2norm * regularization_loss.
+
     - Yaml: ``alpha_l2norm: 0.00001``
     - Argument: ``--alpha-l2norm 0.00001``
     - Default value: ``0.0``
 - **num_ffn_layers_in_input**: Graphstorm provides this argument as an option to increase the size of the parameters in the input layer. This argument will add an MLP layer after computing the input embeddings for each node type. It accepts an integer greater than zero. Generally, `embeds = MLP(embeds)` for each node type in the input layer. If the input is n, it could add n Feedforward neural network layers in the MLP.
+
     - Yaml: ``num_ffn_layers_in_input: 1``
     - Argument: ``--num-ffn-layers-in-input 1``
     - Default value: ``0``
 - **num_ffn_layers_in_gnn**: Graphstorm provides this argument as an option to increase the size of the parameters between gnn layers. This argument will add an MLP layer at the end of each GNN layer. Generally, `h = MLP(h)` between GNN layers in a GNN model. If the input here is n, it could add n feedforward neural network layers here.
+
     - Yaml: ``num_ffn_layers_in_gnn: 1``
     - Argument: ``--num-ffn-layers-in-gnn 1``
     - Default value: ``0``
 - **num_ffn_layers_in_decoder**: Graphstorm provides this argument as an option to increase the size of the parameters in the decoder layer. This argument will add an MLP layer before the last layer of a decoder. If the input here is n, it could add n feedforward neural network layers. Please note, it is only effective when the decoder is an ``MLPEdgeDecoder`` or an ``MLPEFeatEdgeDecoder``. Support for other decoders will be added later.
+
     - Yaml: ``num_ffn_layers_in_decoder: 1``
     - Argument: ``--num-ffn-layers-in-decoder 1``
     - Default value: ``0``
 - **input_activate**: Graphstorm provides this argument as an option to change the activation function in the input layer. Please note, it only accepts 'relu' and 'none'.
+
     - Yaml: ``input_activate: relu``
     - Argument: ``--input-activate relu``
     - Default value: ``none``
 - **gnn_norm**: Graphstorm provides this argument as an option to define the norm type for gnn layers. Please note, it only accepts 'batch' and 'layer' for batchnorm and layernorm respectively.
+
     - Yaml: ``gnn_norm: batch``
     - Argument: ``--gnn-norm batch``
     - Default value: ``none``
@@ -203,18 +238,22 @@ GraphStorm provides a set of parameters to control early stop of training. By de
 Every time evaluation is triggered, GraphStorm checks early stop criteria. For the rounds within early_stop_burnin_rounds evaluation calls, GraphStorm will not use early stop. After early_stop_burnin_rounds, GraphStorm decides if stop early based on the **early_stop_strategy**. There are two strategies: 1) **consecutive_increase**, early stop is triggered if the current validation score is lower than the average of the last **early_stop_rounds** validation scores and 2) **average_increase**, early stop is triggered if for the last **early_stop_rounds** consecutive steps, the validation scores are `decreasing`.
 
 - **early_stop_burnin_rounds**: Burning period calls to start considering early stop.
+
     - Yaml: ``early_stop_burnin_rounds: 100``
     - Argument: ``--early-stop-burnin-rounds 100``
     - Default value: ``0.0``
 - **early_stop_rounds**: The number of rounds for validation scores used to decide if early stop.
+
     - Yaml: ``early_stop_rounds: 5``
     - Argument: ``--early-stop-rounds 5``
     - Default value: ``3.``
 - **early_stop_strategy**: GraphStorm supports two strategies: 1) consecutive_increase and 2) average_increase.
+
     - Yaml: ``early_stop_strategy: consecutive_increase``
     - Argument: ``--early-stop-strategy average_increase``
     - Default value: ``average_increase``
 - **use_early_stop**: Set true to enable early stop.
+
     - Yaml: ``use_early_stop: true``
     - Argument: ``--use-early-stop true``
     - Default value: ``false``
@@ -224,22 +263,27 @@ Model Evaluation Configurations
 GraphStorm provides a set of parameters to control model evaluation.
 
 - **eval_batch_size**: Mini-batch size for computing GNN embeddings in evaluation. You can set eval_batch_size larger than batch_size to speedup GNN embedding computation. To be noted, a larger eval_batch_size will consume more GPU memory.
+
     - Yaml: ``eval_batch_size: 1024``
     - Argument: ``--eval-batch-size 1024``
     - Default value: 10000.
 - **eval_fanout**: (**Required**) The fanout of each GNN layers used in evaluation and inference. It follows the same format as fanout.
+
     - Yaml: ``eval_fanout: "10,10"``
     - Argument: ``--eval-fanout 10,10``
     - Default value: This parameter must be provided by user. But if set the ``--num_layers`` to be 0, which means there is no GNN layer, no need to specify this configuration.
 - **use_mini_batch_infer**: Set true to do mini-batch inference during evaluation and inference. Set false to do full-graph inference during evaluation and inference. For node classification/regression and edge classification/regression tasks, if the evaluation set or testing set is small, mini-batch inference can be more efficient as it does not waste resources to compute node embeddings for nodes not used during inference. However, if the test set is large or the task is link prediction, full graph inference (set use_mini_batch_infer to false) is preferred, as it avoids recomputing node embeddings during inference.
+
     - Yaml: ``use_mini_batch_infer: false``
     - Argument: ``--use-mini-batch-infer false``
     - Default value: ``true``
 - **eval_frequency**: The frequency of doing evaluation. GraphStorm trainers do evaluation at the end of each epoch. However, for large-scale graphs, training one epoch may take hundreds of thousands of iterations. One may want to do evaluations in the middle of an epoch. When eval_frequency is set, every **eval_frequency** iterations, the trainer will do evaluation once. The evaluation results can be printed and reported. See **log_report_frequency** for more details.
+
     - Yaml: ``eval_frequency: 10000``
     - Argument: ``--eval-frequency 10000``
     - Default value: ``sys.maxsize``. The system will not do evaluation.
 - **no_validation**: Set true to avoid do model evaluation (validation) during training.
+
     - Yaml: ``no_validation: true``
     - Argument: ``--no-validation true``
     - Default value: ``false``
@@ -249,18 +293,22 @@ Language Model Specific Configurations
 GraphStorm supports co-training language models with GNN. GraphStorm provides a set of parameters to control language model fine-tuning.
 
 - **lm_tune_lr**: Learning rate for fine-tuning language model.
+
     - Yaml: ``lm_tune_lr: 0.0001``
     - Argument: ``--lm-tune-lr 0.0001``
     - Default value: same as **lr**
 - **lm_train_nodes**: Number of nodes used in LM model fine-tuning for each different LM model.
+
     - Yaml: ``lm_train_nodes: 10``
     - Argument: ``--lm-train-nodes 10``
     - Default value: ``0``
 - **lm_infer_batch_size**: Batch size used in LM model inference.
+
     - Yaml: ``lm_infer_batch_size: 10``
     - Argument: ``--lm-infer-batch-size 10``
     - Default value: ``32``
 - **freeze_lm_encoder_epochs**: Before fine-tuning LM model, how many epochs we will take to warmup a GNN model.
+
     - Yaml: ``freeze_lm_encoder_epochs: 1``
     - Argument: ``--freeze-lm-encoder-epochs 1``
     - Default value: ``0``
@@ -272,10 +320,12 @@ GraphStorm supports node classification, node regression, edge classification, e
 General Configurations
 .........................
 - **task_type**: (**Required**) Supported task type includes node_classification, node_regression, edge_classification, edge_regression, and link_prediction.
+
     - Yaml: ``task_type: node_classification``
     - Argument: ``--task-type node_classification``
     - Default value: This parameter must be provided by user.
-- **eval_metric**: Evaluation metric used during evaluation. The input can be a string specifying the evaluation metric to report or a list of strings specifying a list of evaluation metrics to report. The first evaluation metric is treated as the major metric and is used to choose the best trained model. The supported evaluation metrics of classification tasks include ``accuracy``, ``precision_recall``, ``roc_auc``, ``f1_score``, ``per_class_f1_score``. The supported evaluation metrics of regression tasks include ``rmse``, ``mse`` and ``mae``. The supported evaluation metrics of link prediction tasks include ``mrr``. 
+- **eval_metric**: Evaluation metric used during evaluation. The input can be a string specifying the evaluation metric to report or a list of strings specifying a list of evaluation metrics to report. The first evaluation metric is treated as the major metric and is used to choose the best trained model. The supported evaluation metrics of classification tasks include ``accuracy``, ``precision_recall``, ``roc_auc``, ``f1_score``, ``per_class_f1_score``. The supported evaluation metrics of regression tasks include ``rmse``, ``mse`` and ``mae``. The supported evaluation metrics of link prediction tasks include ``mrr``.
+
     - Yaml: ``eval_metric:``
         | ``- accuracy``
         | ``- precision_recall``
@@ -288,30 +338,37 @@ General Configurations
 Classification and Regression Task
 ......................................
 - **label_field**: (**Required**) The field name of labelled data in the graph data. For node classification tasks, GraphStorm use ``graph.nodes[target_ntype].data[label_field]`` to access node labels. For edge classification tasks, GraphStorm use ``graph.edges[target_etype].data[label_field]`` to access edge labels.
+
     - Yaml: ``label_field: color``
     - Argument: ``--label-field color``
     - Default value: This parameter must be provided by user.
 - **num_classes**: (**Required**) The cardinality of labels in a classification task. Used by node classification and edge classification.
+
     - Yaml: ``num_classes: 10``
     - Argument: ``--num-classes 10``
     - Default value: This parameter must be provided by user.
 - **multilabel**: If set to true, the task is a multi-label classification task. Used by node classification and edge classification.
+
     - Yaml: ``multilabel: true``
     - Argument: ``--multilabel true``
     - Default value: ``false``
 - **multilabel_weights**: Used to specify a weight of positive examples for each class in a multi-label classification task. This is used together with **multilabel**. It is feed into ``torch.nn.BCEWithLogitsLoss`` as ``pos_weight``. The weights should be in the following format `0.1,0.2,0.3,0.1,0.0`. Each field represents the weight of the positive answer for the class n. Suppose there are 3 classes. The multilabel_weights is set to `0.1,0.2,0.3`. Class 0 will have weight of 0.1, class 1 will have weight of 0.2 and class 2 will have weight of 0.3. For more details, see `BCEWithLogitsLoss <https://pytorch.org/docs/stable/generated/torch.nn.BCEWithLogitsLoss.html>`_. If not provided, all classes are treated equally.
+
     - Yaml: ``multilabel_weights: 0.1,0.2,0.3``
     - Argument: ``--multilabel-weights 0.1,0.2,0.3``
     - Default value: ``None``
 - **imbalance_class_weights**: Used to specify a manual rescaling weight given to each class in a single-label multi-class classification task. It is used in imbalanced label use cases. It is feed into torch.nn.CrossEntropyLoss. Each field represents a weight for a class. Suppose there are 3 classes. The imbalance_class_weights is set to `0.1,0.2,0.3`. Class 0 will have weight of 0.1, class 1 will have weight of 0.2 and class 2 will have weight of 0.3. If not provided, all classes are treated equally.
+
     - Yaml: ``imbalance_class_weights: 0.1,0.2,0.3``
     - Argument: ``--imbalance-class-weights 0.1,0.2,0.3``
     - Default value: ``None``
 - **return_proba**: For classification task, this configuration determines whether to return probability estimates for each class or the maximum probable class. Set `true`` to return probability estimates and `false` to return the maximum probable class.
+
     - Yaml: ``return_proba: true``
     - Argument: ``--return_proba true``
     - Default value: ``true``
 - **save_prediction_path**: Path to save prediction results. This is used in node/edge classification/regression inference.
+
     - Yaml: ``save_prediction_path: /data/infer-output/predictions/``
     - Argument: ``--save-prediction-path /data/infer-output/predictions/``
     - Default value: If not provided, it will be the same as save_embed_path.
@@ -319,6 +376,7 @@ Classification and Regression Task
 Node Classification/Regression Specific
 .........................................
 - **target_ntype**: (**Required**) The node type for prediction.
+
     - Yaml: ``target_ntype: movie``
     - Argument: ``--target-ntype movie``
     - Default value: This parameter must be provided by user.
@@ -326,29 +384,35 @@ Node Classification/Regression Specific
 Edge Classification/Regression Specific
 ..........................................
 - **target_etype**: (**Required**) The list of canonical edge types that will be added as a training target in edge classification/regression tasks, for example ``--train-etype query,clicks,asin`` or ``--train-etype query,clicks,asin query,search,asin``. A canonical edge type should be formatted as `src_node_type,relation_type,dst_node_type`. Currently, GraphStorm only supports single task edge classification/regression, i.e., it only accepts one canonical edge type.
+
     - Yaml: ``target_etype:``
            | ``- query,clicks,asin``
     - Argument: ``--target-etype query,clicks,asin``
     - Default value: This parameter must be provided by user.
 - **remove_target_edge_type**: When set to true, GraphStorm removes target_etype in message passing, i.e., any edge with target_etype will not be sampled during training and inference.
+
     - Yaml: ``remove_target_edge_type: false``
     - Argument: ``--remove-target-edge-type false``
     - Default value: ``true``
 - **reverse_edge_types_map**: A list of reverse edge type info. Each edge type is in the following format: `head,relation,reverse_relation,tail`. For example: `["query,adds,rev-adds,asin", "query,clicks,rev-clicks,asin"]`. For edge classification/regression tasks, if **remove_target_edge_type** is set true and **reverse_edge_type_map** is provided, GraphStorm will remove both **target_etype** and the corresponding reverse edge type(s) in message passing. In certain cases, any edge with **target_etype** or reverse **target_etype** will not be sampled during training and inference. For link prediction tasks, if **exclude_training_targets** is set to ``true`` and **reverse_edge_type_map** is provided, GraphStorm will remove both target edges with **train_etype** and the corresponding reverse edges with the reverse edge types of **train_etype** in message passing. In contrast to edge classification/regression tasks, for link prediction tasks, GraphStorm only excludes specific edges instead of all edges with **target_etype** or reverse **target_etype** in message passing.
+
     - Yaml: ``reverse_edge_types_map:``
            | ``- query,adds,rev-adds,asin``
            | ``- query,clicks,rev-clicks,asin``
     - Argument: ``--reverse-edge-types-map query,adds,rev-adds,asin query,clicks,rev-clicks,asin``
     - Default value: ``None``
 - **decoder_type**: Type of edge classification or regression decoder. Built-in decoders include ``DenseBiDecoder`` and ``MLPDecoder``. ``DenseBiDecoder`` implements the bi-linear decoder used in GCMC. ``MLPEdgeDecoder`` simply applies Multilayer Perceptron layers for prediction.
+
     - Yaml: ``decoder-type: DenseBiDecoder``
     - Argument: ``--decoder-type MLPDecoder``
     - Default value: ``DenseBiDecoder``
 - **num_decoder_basis**: The number of basis for DenseBiDecoder in edge prediction task.
+
     - Yaml: ``num_decoder_basis: 2``
     - Argument: ``--num-decoder-basis 2``
     - Default value: ``2``
 - **decoder_edge_feat**: A list of edge features that can be used by a decoder to enhance its performance.
+
     - Yaml: ``decoder_edge_feat:``
             | ``- "fname"``
         Or
@@ -360,51 +424,62 @@ Edge Classification/Regression Specific
 Link Prediction Task
 ......................
 - **train_etype**: The list of canonical edge type that will be added as training target with the target edge type(s). If not provided, all edge types will be used as training target. A canonical edge type should be formatted as `src_node_type,relation_type,dst_node_type`.
+
     - Yaml: ``train_etype:``
            | ``- query,clicks,asin``
            | ``- query,adds,asin``
     - Argument: ``--train-etype query,clicks,asin query,adds,asin``
     - Default value: ``None``
 - **eval_etype**: The list of canonical edge type that will be added as evaluation target with the target edge type(s). If not provided, all edge types will be used as evaluation target. In some link prediction use cases, users want to train a model using all edges of a graph but only do link prediction on specific edge type(s) for downstream applications. In certain cases, they only care about the model performance on specific edge types.
+
     - Yaml: ``eval_etype:``
            | ``- query,clicks,asin``
            | ``- query,adds,asin``
     - Argument: ``--eval-etype query,clicks,asin query,adds,asin``
     - Default value: ``None``
 - **exclude_training_targets**: If it is set to ``true``, GraphStorm removes the training targets from the GNN computation graph. If true, **reverse_edge_types_map** **MUST** be provided.
+
     - Yaml: ``exclude_training_targets: false``
     - Argument: ``--exclude-training-targets false``
     - Default value: ``true``
 - **train_negative_sampler**: The negative sampler used for link prediction training. Built-in samplers include ``uniform``, ``joint``, ``localuniform``, ``all_etype_uniform`` and ``all_etype_joint``.
+
     - Yaml: ``train_negative_sampler: uniform``
     - Argument: ``--train-negative-sampler joint``
     - Default value: ``uniform``
 - **eval_negative_sampler**: The negative sampler used for link prediction testing and evaluation. Built-in samplers include ``uniform``, ``joint``, ``localuniform``, ``all_etype_uniform`` and ``all_etype_joint``.
+
     - Yaml: ``eval_negative_sampler: uniform``
     - Argument: ``--eval-negative-sampler joint``
     - Default value: ``joint``
 - **num_negative_edges**: Number of negative edges sampled for each positive edge during training.
+
     - Yaml: ``num_negative_edges: 32``
     - Argument: ``--num-negative-edges 32``
     - Default value: ``16``
 - **num_negative_edges_eval**: Number of negative edges sampled for each positive edge in the validation and test set.
+
     - Yaml: ``num_negative_edges_eval: 1000``
     - Argument: ``--num-negative-edges-eval 1000``
     - Default value: ``1000``
 - **lp_decoder_type**: Set the decoder type for loss function in Link Prediction tasks. Currently GraphStorm support  ``dot_product`` and ``DistMult``.
+
     - Yaml: ``lp_decoder_type: dot_product``
     - Argument: ``--lp-decoder-type dot_product``
     - Default value: ``dot_product``
 - **gamma**: Gamma for ``DistMult``. The margin value in the score function.
+
     - Yaml: ``gamma: 10.0``
     - Argument: ``--gamma 10.0``
     - Default value: ``12.0``
 - **lp_loss_func**: Link prediction loss function. Builtin loss functions include ``cross_entropy`` and ``logsigmoid``.
+
     - Yaml: ``lp_loss_func: cross_entropy``
     - Argument: ``--lp-loss-func logsigmoid``
     - Default value: ``cross_entropy``
 
 - **lp_edge_weight_for_loss**: Edge feature field name for edge weight. The edge weight is used to rescale the positive edge loss for link prediction tasks.
+
     - Yaml: ``lp_edge_weight_for_loss:``
                 | ``- "weight"``
         Or
@@ -419,14 +494,17 @@ Distillation Specific Configurations
 GraphStorm provides a set of parameters to control GNN distillation.
 
 - **textual_data_path**: The path to load the textual data for distillation. User need to specify a path of directory with two sub-directory for ``train`` split and ``val`` split. In each split, there can be one or more ``*.parquet file``s. Find more details in ``training_scripts/gsgnn_dt/README.md``.
+
     - Yaml: ``textual_data_path: <str>``
     - Argument: ``--textual-data-path <str>``
     - Default value: This parameter must be provided by user.
 - **max_distill_step**: The maximum of training step for each node type for distillation.
+
     - Yaml: ``max_distill_step: 10000``
     - Argument: ``--max-distill-step 10000``
     - Default value: ``10000``
 - **max_seq_len**: The maximum sequence length for tokenized textual data.
+
     - Yaml: ``max_seq_len: 1024``
     - Argument: ``--max-seq-len 1024``
     - Default value: ``1024``

From d0fe353f85319b698e16c81c885db7307a3a678d Mon Sep 17 00:00:00 2001
From: Zichen Wang <zichewan@amazon.com>
Date: Tue, 10 Oct 2023 20:23:58 -0400
Subject: [PATCH 3/6] Bug fix dist model save (#537)

*Issue #, if available:* #535

*Description of changes:*


By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice.

---------

Co-authored-by: Da Zheng <zhengda1936@gmail.com>
---
 python/graphstorm/trainer/gsgnn_trainer.py | 7 ++++---
 tests/end2end-tests/graphstorm-nc/test.sh  | 4 +++-
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/python/graphstorm/trainer/gsgnn_trainer.py b/python/graphstorm/trainer/gsgnn_trainer.py
index 90b656c9f9..c173d4ed40 100644
--- a/python/graphstorm/trainer/gsgnn_trainer.py
+++ b/python/graphstorm/trainer/gsgnn_trainer.py
@@ -26,7 +26,7 @@
 from ..model.utils import save_model_results_json
 from ..config import GRAPHSTORM_MODEL_ALL_LAYERS
 
-from ..utils import barrier, get_rank
+from ..utils import barrier, get_rank, is_distributed
 
 class GSgnnTrainer():
     """ Generic GSgnn trainer.
@@ -192,11 +192,12 @@ def save_model(self, model, epoch, i, save_model_path):
         '''
         barrier()
         if save_model_path is not None:
-            assert isinstance(model.module, (GSgnnModel, GSgnnModelBase)), \
+            module = model.module if is_distributed() else model
+            assert isinstance(module, (GSgnnModel, GSgnnModelBase)), \
                 "Please make sure the model derives from GSgnnModel or GSgnnModelBase, " \
                 "which provides a scalable model saving implementation."
             save_model_path = self._gen_model_path(save_model_path, epoch, i)
-            model.module.save_model(save_model_path)
+            module.save_model(save_model_path)
             self.optimizer.save_opt_state(save_model_path)
 
         # make sure each trainer finishes its own model saving task.
diff --git a/tests/end2end-tests/graphstorm-nc/test.sh b/tests/end2end-tests/graphstorm-nc/test.sh
index a7e68aa761..8396f8c05d 100644
--- a/tests/end2end-tests/graphstorm-nc/test.sh
+++ b/tests/end2end-tests/graphstorm-nc/test.sh
@@ -28,10 +28,12 @@ echo "Test GraphStorm node classification"
 date
 
 echo "**************standalone"
-python3 $GS_HOME/python/graphstorm/run/gsgnn_np/gsgnn_np.py --part-config /data/movielen_100k_train_val_1p_4t/movie-lens-100k.json --cf $GS_HOME/training_scripts/gsgnn_np/ml_nc.yaml
+python3 $GS_HOME/python/graphstorm/run/gsgnn_np/gsgnn_np.py --part-config /data/movielen_100k_train_val_1p_4t/movie-lens-100k.json --cf $GS_HOME/training_scripts/gsgnn_np/ml_nc.yaml --save-model-path ./models/movielen_100k/train_val/standalone
 
 error_and_exit $?
 
+rm -R ./models/movielen_100k/train_val/standalone
+
 echo "**************dataset: MovieLens, RGCN layer: 1, node feat: fixed HF BERT, BERT nodes: movie, inference: mini-batch"
 python3 -m graphstorm.run.gs_node_classification --workspace $GS_HOME/training_scripts/gsgnn_np/ --num-trainers $NUM_TRAINERS --num-servers 1 --num-samplers 0 --part-config /data/movielen_100k_train_val_1p_4t/movie-lens-100k.json --ip-config ip_list.txt --ssh-port 2222 --cf ml_nc.yaml
 

From 4e1fbe26b813a5a8efdf2d2b5a6445ae4dfb4b9f Mon Sep 17 00:00:00 2001
From: "xiang song(charlie.song)" <classicxsong@gmail.com>
Date: Tue, 10 Oct 2023 18:48:36 -0700
Subject: [PATCH 4/6] Allow Sagemaker training jobs resume an existing
 checkpoint (#538)

*Issue #, if available:*
#450

*Description of changes:*


By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice.

---------

Co-authored-by: Xiang Song <xiangsx@amazon.com>
---
 .github/workflow_scripts/lint_check.sh        |  1 +
 docs/source/scale/sagemaker.rst               |  5 ++++
 .../graphstorm/sagemaker/sagemaker_train.py   | 23 ++++++++++++++++---
 sagemaker/README.md                           |  5 ++++
 sagemaker/launch/launch_train.py              |  9 ++++++--
 sagemaker/run/train_entry.py                  |  3 +++
 6 files changed, 41 insertions(+), 5 deletions(-)

diff --git a/.github/workflow_scripts/lint_check.sh b/.github/workflow_scripts/lint_check.sh
index 3eecaf93c3..c599ec3c13 100644
--- a/.github/workflow_scripts/lint_check.sh
+++ b/.github/workflow_scripts/lint_check.sh
@@ -4,6 +4,7 @@ cd ../../
 set -ex
 
 python3 -m pip install --upgrade prospector pip
+yes | pip3 install astroid==v3.0.0
 FORCE_CUDA=1 python3 -m pip install -e '.[test]'  --no-build-isolation
 pylint --rcfile=./tests/lint/pylintrc ./python/graphstorm/data/*.py
 pylint --rcfile=./tests/lint/pylintrc ./python/graphstorm/dataloading/
diff --git a/docs/source/scale/sagemaker.rst b/docs/source/scale/sagemaker.rst
index 32ccffb809..93283246b9 100644
--- a/docs/source/scale/sagemaker.rst
+++ b/docs/source/scale/sagemaker.rst
@@ -160,6 +160,11 @@ Because we are using a three-partition OGB-MAG graph, we need to set the ``--ins
 
 The trained model artifact will be stored in the S3 location provided through the ``--model-artifact-s3`` argument. You can use the following command to check the model artifacts after the training completes.
 
+If you want to resume a saved model checkpoint to do model fine-tuning you can pass
+the S3 address of the model checkpoint through the ``--model-checkpoint-to-load``
+argument. For example by passing ``--model-checkpoint-to-load s3://mag-model/epoch-2/``,
+GraphStorm will initialize the model parameters with the model checkpoint stored in ``s3://mag-model/epoch-2/``.
+
 .. code-block:: bash
 
     aws s3 ls s3://<PATH_TO_SAVE_TRAINED_MODEL>/
diff --git a/python/graphstorm/sagemaker/sagemaker_train.py b/python/graphstorm/sagemaker/sagemaker_train.py
index 6c92d7a88a..3e282774a2 100644
--- a/python/graphstorm/sagemaker/sagemaker_train.py
+++ b/python/graphstorm/sagemaker/sagemaker_train.py
@@ -39,11 +39,12 @@
                     barrier,
                     terminate_workers,
                     wait_for_exit,
+                    download_model,
                     upload_model_artifacts)
 
 def launch_train_task(task_type, num_gpus, graph_config,
     save_model_path, ip_list, yaml_path,
-    extra_args, state_q, custom_script):
+    extra_args, state_q, custom_script, restore_model_path=None):
     """ Launch SageMaker training task
 
     Parameters
@@ -68,6 +69,10 @@ def launch_train_task(task_type, num_gpus, graph_config,
         A queue used to return execution result (success or failure)
     custom_script: str
         Custom training script provided by a customer to run customer training logic.
+    restore_model_path: str
+        Path for the model to restore for model fine-tuning.
+        Default: None
+
     Return
     ------
     Thread: training task thread
@@ -97,7 +102,10 @@ def launch_train_task(task_type, num_gpus, graph_config,
         "--ssh-port", "22"]
     launch_cmd += [custom_script] if custom_script is not None else []
     launch_cmd += ["--cf", f"{yaml_path}",
-        "--save-model-path", f"{save_model_path}"] + extra_args
+        "--save-model-path", f"{save_model_path}"] + \
+        ["--restore-model-path", f"{restore_model_path}"] \
+            if restore_model_path is not None else []
+    launch_cmd += extra_args
 
     print(launch_cmd)
 
@@ -145,7 +153,9 @@ def run_train(args, unknownargs):
     """
     num_gpus = args.num_gpus
     data_path = args.data_path
+    restore_model_path = "/tmp/gsgnn_model_checkpoint/"
     output_path = "/tmp/gsgnn_model/"
+    os.makedirs(restore_model_path, exist_ok=True)
     os.makedirs(output_path, exist_ok=True)
 
     # start the ssh server
@@ -204,6 +214,7 @@ def run_train(args, unknownargs):
     graph_data_s3 = args.graph_data_s3
     task_type = args.task_type
     train_yaml_s3 = args.train_yaml_s3
+    model_checkpoint_s3 = args.model_checkpoint_to_load
     model_artifact_s3 = args.model_artifact_s3.rstrip('/')
     custom_script = args.custom_script
 
@@ -213,6 +224,11 @@ def run_train(args, unknownargs):
         data_path, sagemaker_session)
     graph_config_path = download_graph(graph_data_s3, graph_name,
         host_rank, data_path, sagemaker_session)
+    if model_checkpoint_s3 is not None:
+        # Download Saved model checkpoint to resume
+        download_model(model_checkpoint_s3, restore_model_path, sagemaker_session)
+        print(f"{restore_model_path} {os.listdir(restore_model_path)}")
+
 
     err_code = 0
     if host_rank == 0:
@@ -236,7 +252,8 @@ def run_train(args, unknownargs):
                                             yaml_path,
                                             gs_params,
                                             state_q,
-                                            custom_script)
+                                            custom_script,
+                                            restore_model_path)
             train_task.join()
             err_code = state_q.get()
         except RuntimeError as e:
diff --git a/sagemaker/README.md b/sagemaker/README.md
index 3952d96048..c26f606a91 100644
--- a/sagemaker/README.md
+++ b/sagemaker/README.md
@@ -137,6 +137,11 @@ You can use following command to check the model artifacts:
 aws s3 ls s3://PATH_TO_SAVE_TRAINED_MODEL/
 ```
 
+If you want to resume a saved model checkpoint to do model fine-tuning you can pass
+the S3 address of the model checkpoint through the ``--model-checkpoint-to-load``
+argument. For example by passing ``--model-checkpoint-to-load s3://mag-model/epoch-2/``,
+GraphStorm will initialize the model parameters with the model checkpoint stored in ``s3://mag-model/epoch-2/``.
+
 Please note `save_embed_path` and `save_prediction_path` must be disabled, i.e., set to 'None' when using SageMaker.
 They only work with shared file system while SageMaker solution does not support using shared file system now.
 
diff --git a/sagemaker/launch/launch_train.py b/sagemaker/launch/launch_train.py
index e80143bdc7..65d2db4020 100644
--- a/sagemaker/launch/launch_train.py
+++ b/sagemaker/launch/launch_train.py
@@ -50,6 +50,7 @@ def run_job(input_args, image, unknowargs):
     graph_data_s3 = input_args.graph_data_s3 # S3 location storing partitioned graph data
     train_yaml_s3 = input_args.yaml_s3 # S3 location storing the yaml file
     model_artifact_s3 = input_args.model_artifact_s3 # Where to store model artifacts
+    model_checkpoint_to_load = input_args.model_checkpoint_to_load # S3 location of a saved model.
     custom_script = input_args.custom_script # custom_script if any
 
     boto_session = boto3.session.Session(region_name=region)
@@ -65,7 +66,8 @@ def run_job(input_args, image, unknowargs):
               "graph-name": graph_name,
               "graph-data-s3": graph_data_s3,
               "train-yaml-s3": train_yaml_s3,
-              "model-artifact-s3": model_artifact_s3}
+              "model-artifact-s3": model_artifact_s3,
+              "model-checkpoint-to-load": model_checkpoint_to_load}
     if custom_script is not None:
         params["custom-script"] = custom_script
     # We must handle cases like
@@ -133,7 +135,10 @@ def get_train_parser():
         help="S3 location of training yaml file. "
              "Do not store it with partitioned graph", required=True)
     training_args.add_argument("--model-artifact-s3", type=str, default=None,
-        help="S3 bucket to save model artifacts")
+        help="S3 path to save model artifacts")
+    training_args.add_argument("--model-checkpoint-to-load", type=str, default=None,
+        help="S3 path to a model checkpoint from a previous training task "
+             "that is going to be resumed.")
     training_args.add_argument("--custom-script", type=str, default=None,
         help="Custom training script provided by a customer to run customer training logic. \
             Please provide the path of the script within the docker image")
diff --git a/sagemaker/run/train_entry.py b/sagemaker/run/train_entry.py
index 7a545e7dc3..a6af59f7af 100644
--- a/sagemaker/run/train_entry.py
+++ b/sagemaker/run/train_entry.py
@@ -39,6 +39,9 @@ def parse_train_args():
              "Do not store it with partitioned graph")
     parser.add_argument("--model-artifact-s3", type=str,
         help="S3 location to store the model artifacts.")
+    parser.add_argument("--model-checkpoint-to-load", type=str, default=None,
+        help="S3 path to a model checkpoint from a previous training task "
+             "that is going to be resumed.")
     parser.add_argument("--custom-script", type=str, default=None,
         help="Custom training script provided by a customer to run customer training logic. \
             Please provide the path of the script within the docker image")

From 5667576b11c90fb3c584e9d5f2719f5309534e06 Mon Sep 17 00:00:00 2001
From: Zichen Wang <zichewan@amazon.com>
Date: Wed, 11 Oct 2023 16:54:22 -0400
Subject: [PATCH 5/6] [Documentation] Standalone mode demo notebook (#530)

*Issue #, if available:*

*Description of changes:* Added Jupyter notebook to demonstrate
standalone mode.


By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice.

---------

Co-authored-by: Da Zheng <zhengda1936@gmail.com>
---
 examples/standalone_mode_demo.ipynb | 570 ++++++++++++++++++++++++++++
 python/graphstorm/eval/evaluator.py |   5 +
 2 files changed, 575 insertions(+)
 create mode 100644 examples/standalone_mode_demo.ipynb

diff --git a/examples/standalone_mode_demo.ipynb b/examples/standalone_mode_demo.ipynb
new file mode 100644
index 0000000000..bf7203e7b6
--- /dev/null
+++ b/examples/standalone_mode_demo.ipynb
@@ -0,0 +1,570 @@
+{
+ "cells": [
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Graphstorm Standalone Mode Demonstration\n",
+    "\n",
+    "In this notebook, we'll demonstrate how to use the standalone mode of Graphstorm. The standalone mode is primarily designed for model developers to prototype their data and model training/evaluation/inference pipelines using a single machine. \n",
+    "\n",
+    "---\n",
+    "\n",
+    "## Setup \n",
+    "\n",
+    "This notebook requires installing graphstorm using pip. Please find [more details on installation of graphstorm](https://graphstorm.readthedocs.io/en/latest/install/env-setup.html#setup-graphstorm-with-pip-packages)."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# !pip install git+https://github.com/awslabs/graphstorm"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import matplotlib.pyplot as plt\n",
+    "# import graphstorm related stuffs\n",
+    "import graphstorm as gs\n",
+    "from graphstorm.trainer import GSgnnNodePredictionTrainer\n",
+    "from graphstorm.dataloading import GSgnnNodeTrainData, GSgnnNodeDataLoader, GSgnnNodeInferData\n",
+    "from graphstorm.model import GSgnnNodeModel, GSNodeEncoderInputLayer, EntityClassifier, ClassifyLossFunc, RelationalGCNEncoder\n",
+    "from graphstorm.inference import GSgnnNodePredictionInferrer\n",
+    "from graphstorm.eval import GSgnnAccEvaluator"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# initialize the graphstorm standalone environment\n",
+    "gs.initialize(ip_config=None, backend='gloo')\n",
+    "device = gs.utils.setup_device(0)"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Develop a GNN model for node-classification \n",
+    "\n",
+    "Next, we use a GNN model to go over the model development cycle of a node classification problem on the MovieLens dataset. \n",
+    "\n",
+    "\n",
+    "### 0. Load the constructed graph data\n",
+    "\n",
+    "In this demo we use the MovieLens graph constructed by [graphstorm's processing pipeline](https://graphstorm.readthedocs.io/en/latest/gs-processing/usage/example.html)."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# constructed graphstorm data config:\n",
+    "constructed_graph_config = '/mnt/efs/gsf-data/movielen_100k_train_val_1p_4t/movie-lens-100k.json'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# config about the dataset and task:\n",
+    "graph_name = 'movie-lens-100k'\n",
+    "target_ntype = 'movie'\n",
+    "node_feat_name = None\n",
+    "feat_sizes = {'movie': 0, 'user': 0}\n",
+    "label_field = 'label'\n",
+    "num_classes = 19\n",
+    "multilabel = False\n",
+    "\n",
+    "# learning params:\n",
+    "batch_size = 128\n",
+    "lr = 0.001\n",
+    "sparse_optimizer_lr = 0.01\n",
+    "weight_decay = 0.\n",
+    "\n",
+    "# model architecture:\n",
+    "hidden_size = 128\n",
+    "fanout = [4]\n",
+    "dropout = 0.0\n",
+    "num_bases = -1\n",
+    "num_layers = 1\n",
+    "use_self_loop = True"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# load the constructed graph data\n",
+    "train_data = GSgnnNodeTrainData(\n",
+    "    graph_name=graph_name,\n",
+    "    part_config=constructed_graph_config,\n",
+    "    train_ntypes=target_ntype,\n",
+    "    eval_ntypes=target_ntype,\n",
+    "    node_feat_field=node_feat_name,\n",
+    "    label_field=label_field)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Number of nodes:\n",
+      "- movie: 1682\n",
+      "- user: 943\n",
+      "Number of edges:\n",
+      "- rating-rev: 100000\n",
+      "- rating: 100000\n"
+     ]
+    }
+   ],
+   "source": [
+    "# train_data.g stores a DistGraph object\n",
+    "print('Number of nodes:')\n",
+    "for ntype in train_data.g.ntypes:\n",
+    "    print(f'- {ntype}: {train_data.g.num_nodes(ntype)}')\n",
+    "print('Number of edges:')\n",
+    "for etype in train_data.g.etypes:\n",
+    "    print(f'- {etype}: {train_data.g.num_edges(etype)}')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# set up data loaders\n",
+    "dataloader = GSgnnNodeDataLoader(\n",
+    "    train_data, train_data.train_idxs, fanout=fanout,\n",
+    "    batch_size=batch_size,\n",
+    "    device=device, train_task=True)\n",
+    "\n",
+    "val_dataloader = GSgnnNodeDataLoader(\n",
+    "    train_data, train_data.val_idxs, fanout=fanout,\n",
+    "    batch_size=batch_size,\n",
+    "    device=device, train_task=False)\n",
+    "\n",
+    "test_dataloader = GSgnnNodeDataLoader(\n",
+    "    train_data, train_data.test_idxs, fanout=fanout,\n",
+    "    batch_size=batch_size,\n",
+    "    device=device, train_task=False)"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### 2. Create a GNN model\n",
+    "\n",
+    "A GSF model should contain the following components: \n",
+    "- Input encoder for nodes (and optionally edges): process and project input features and embeddings into a certain dimension\n",
+    "- GNN encoder: performs message-passing on projected node/edge inputs\n",
+    "- Decoder: specific for tasks on the graph\n",
+    "\n",
+    "We can see the following codes set up a `GSgnnNodeModel` model composed of `GSNodeEncoderInputLayer`, `RelationalGCNEncoder`, `EntityClassifier` step-by-step. One can also replace individual components/layers with a custom model for development purpose."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "GSgnnNodeModel(\n",
+       "  (_node_input_encoder): GSNodeEncoderInputLayer(\n",
+       "    (dropout): Dropout(p=0.0, inplace=False)\n",
+       "    (proj_matrix): ParameterDict(\n",
+       "        (movie): Parameter containing: [torch.FloatTensor of size 128x128]\n",
+       "        (user): Parameter containing: [torch.FloatTensor of size 128x128]\n",
+       "    )\n",
+       "    (input_projs): ParameterDict()\n",
+       "    (ngnn_mlp): ModuleDict(\n",
+       "      (movie): NGNNMLP(\n",
+       "        (dropout): Dropout(p=0.0, inplace=False)\n",
+       "        (ngnn_gnn): ParameterList()\n",
+       "      )\n",
+       "      (user): NGNNMLP(\n",
+       "        (dropout): Dropout(p=0.0, inplace=False)\n",
+       "        (ngnn_gnn): ParameterList()\n",
+       "      )\n",
+       "    )\n",
+       "  )\n",
+       "  (_gnn_encoder): RelationalGCNEncoder(\n",
+       "    (_layers): ModuleList(\n",
+       "      (0): RelGraphConvLayer(\n",
+       "        (conv): HeteroGraphConv(\n",
+       "          (mods): ModuleDict(\n",
+       "            (('movie', 'rating-rev', 'user')): GraphConv(in=128, out=128, normalization=right, activation=None)\n",
+       "            (('user', 'rating', 'movie')): GraphConv(in=128, out=128, normalization=right, activation=None)\n",
+       "          )\n",
+       "        )\n",
+       "        (ngnn_mlp): NGNNMLP(\n",
+       "          (dropout): Dropout(p=0.0, inplace=False)\n",
+       "          (ngnn_gnn): ParameterList()\n",
+       "        )\n",
+       "        (dropout): Dropout(p=0.0, inplace=False)\n",
+       "      )\n",
+       "    )\n",
+       "  )\n",
+       "  (_decoder): EntityClassifier(\n",
+       "    (dropout): Dropout(p=0, inplace=False)\n",
+       "  )\n",
+       "  (_loss_fn): ClassifyLossFunc(\n",
+       "    (loss_fn): CrossEntropyLoss()\n",
+       "  )\n",
+       ")"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# create a gsf model\n",
+    "model = GSgnnNodeModel(alpha_l2norm=0.)\n",
+    "\n",
+    "# set input layer encoder\n",
+    "encoder = GSNodeEncoderInputLayer(\n",
+    "    train_data.g, \n",
+    "    feat_sizes,\n",
+    "    hidden_size,\n",
+    "    dropout=dropout,\n",
+    "    activation=None)\n",
+    "model.set_node_input_encoder(encoder)\n",
+    "\n",
+    "# set GNN encoder\n",
+    "gnn_encoder = RelationalGCNEncoder(\n",
+    "    train_data.g,\n",
+    "    hidden_size, hidden_size,\n",
+    "    num_bases=num_bases,\n",
+    "    num_hidden_layers=num_layers - 1,\n",
+    "    dropout=dropout,\n",
+    "    use_self_loop=use_self_loop)\n",
+    "model.set_gnn_encoder(gnn_encoder)\n",
+    "\n",
+    "# set decoder specific to node-classification task\n",
+    "model.set_decoder(EntityClassifier(\n",
+    "    model.node_input_encoder.out_dims,\n",
+    "    num_classes, \n",
+    "    multilabel))\n",
+    "# classification loss function\n",
+    "model.set_loss_func(ClassifyLossFunc(multilabel))\n",
+    "\n",
+    "model.init_optimizer(\n",
+    "    lr=lr, \n",
+    "    sparse_optimizer_lr=sparse_optimizer_lr,\n",
+    "    weight_decay=weight_decay\n",
+    "    )\n",
+    "\n",
+    "model"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### 3. Create a graphstorm trainer and train the model\n",
+    "The trainers is task-specific in graphstorm. It handles:\n",
+    "1. model training/evaluation loops\n",
+    "2. saving and restoring model checkpoints\n",
+    "3. early-stopping"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# create a trainer for the model\n",
+    "trainer = GSgnnNodePredictionTrainer(\n",
+    "    model, \n",
+    "    topk_model_to_save=1)\n",
+    "\n",
+    "# set up device for the trainer\n",
+    "trainer.setup_device(device=device)\n",
+    "\n",
+    "# set up evaluator for the trainer:\n",
+    "evaluator = GSgnnAccEvaluator(\n",
+    "    eval_frequency=10000,\n",
+    "    eval_metric=['accuracy'],\n",
+    "    multilabel=multilabel)\n",
+    "\n",
+    "trainer.setup_evaluator(evaluator)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "WARNING:root:We do not export the state of sparse optimizer\n",
+      "WARNING:root:We do not export the state of sparse optimizer\n",
+      "WARNING:root:We do not export the state of sparse optimizer\n",
+      "WARNING:root:We do not export the state of sparse optimizer\n",
+      "WARNING:root:We do not export the state of sparse optimizer\n",
+      "WARNING:root:We do not export the state of sparse optimizer\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Train the model with the trainer\n",
+    "trainer.fit(\n",
+    "    train_loader=dataloader, \n",
+    "    val_loader=val_dataloader,\n",
+    "    test_loader=test_dataloader, \n",
+    "    num_epochs=10,\n",
+    "    save_model_path='nc_model/',\n",
+    ")"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Next, we examine the model performance over the training process"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Extract accuracies from the trainer's evaluator:\n",
+    "val_accs, test_accs = [], []\n",
+    "for val_acc, test_acc in trainer.evaluator.history:\n",
+    "    val_accs.append(val_acc['accuracy'])\n",
+    "    test_accs.append(test_acc['accuracy'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkAAAAGzCAYAAADHdKgcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB0rElEQVR4nO3dd3hU1dbH8e/MpPeEVCAkofcACSU0C9K7SgcpdlFA1PfKtVz1esFyVRQF26WoEKoICkqR3iEk9CIlhZAQAqSTNnPeP06IRloSkpzMzPo8zzxMTmbO/IZAsrLP2nvrFEVREEIIIYSwInqtAwghhBBCVDUpgIQQQghhdaQAEkIIIYTVkQJICCGEEFZHCiAhhBBCWB0pgIQQQghhdaQAEkIIIYTVkQJICCGEEFZHCiAhhBBCWB0pgIQQQghhdWy0DjB79mw+/PBDkpKSaNasGTNnzqRLly63fOyOHTv4xz/+wcmTJ8nJySEoKIinn36aF198scTjVqxYwRtvvMHZs2epV68e//nPfxg8eHCpM5lMJi5evIirqys6ne6e3p8QQgghqoaiKGRmZlKzZk30+ruM8SgaWrx4sWJra6t88803yvHjx5XJkycrzs7OSlxc3C0ff/DgQWXRokXK0aNHlfPnzyvff/+94uTkpHz11VfFj9m1a5diMBiU6dOnKydOnFCmT5+u2NjYKHv27Cl1roSEBAWQm9zkJje5yU1uZnhLSEi46896naJotxlq+/btadOmDXPmzCk+1qRJEwYNGsSMGTNKdY6HH34YZ2dnvv/+ewCGDRtGRkYGv/76a/FjevXqhaenJ5GRkaU6Z3p6Oh4eHiQkJODm5laGdySEEEIIrWRkZBAYGEhaWhru7u53fKxml8Dy8/OJiori1VdfLXG8R48e7Nq1q1TniI6OZteuXbz77rvFx3bv3n3TJbGePXsyc+bM254nLy+PvLy84o8zMzMBcHNzkwJICCGEMDOlaV/RrAk6NTUVo9GIn59fieN+fn4kJyff8bm1a9fG3t6e8PBwJk6cyBNPPFH8ueTk5DKfc8aMGbi7uxffAgMDy/GOhBBCCGEuNJ8F9vcqTVGUu1Zu27dv58CBA3z55ZfMnDnzpktbZT3ntGnTSE9PL74lJCSU8V0IIYQQwpxodgnM29sbg8Fw08hMSkrKTSM4fxcSEgJAixYtuHTpEm+99RYjRowAwN/fv8zntLe3x97evjxvQwghhBBmSLMCyM7OjrCwMDZs2FBiivqGDRsYOHBgqc+jKEqJ/p2IiAg2bNhQog9o/fr1dOzYsWKC/4XRaKSgoKDCz2tJbG1tMRgMWscQQgghStB0HaCpU6cyZswYwsPDiYiI4OuvvyY+Pp5nnnkGUC9NJSYm8t133wHwxRdfUKdOHRo3bgyo6wL997//5YUXXig+5+TJk+natSvvv/8+AwcOZNWqVWzcuJEdO3ZUWG5FUUhOTiYtLa3CzmnJPDw88Pf3lzWVhBBCVBuaFkDDhg3jypUrvPPOOyQlJdG8eXPWrl1LUFAQAElJScTHxxc/3mQyMW3aNM6fP4+NjQ316tXjvffe4+mnny5+TMeOHVm8eDGvv/46b7zxBvXq1WPJkiW0b9++wnLfKH58fX1xcnKSH+y3oSgKOTk5pKSkABAQEKBxIiGEEEKl6TpA1VVGRgbu7u6kp6ffNA3eaDRy+vRpfH19qVGjhkYJzcuVK1dISUmhYcOGcjlMCCFEpbnTz++/03wWmLm50fPj5OSkcRLzcePvSvqlhBBCVBdSAJWTXPYqPfm7EkIIUd1IASSEEEIIqyMFkCi14ODgO24pIoQQQpgLKYCEEEIIYXWkABJCCHFb+YUmrSMIUSmkALISX331FbVq1cJkKvnNbMCAAYwdO5azZ88ycOBA/Pz8cHFxoW3btmzcuFGjtEKI6mDujvM0efM33lp9TAohYXGkAKoAiqKQk1+oya20yzgNGTKE1NRUNm/eXHzs2rVrrFu3jlGjRpGVlUWfPn3YuHEj0dHR9OzZk/79+5dYiFIIYT2u5xv5bNMfGE0K83fFMvSr3SSmXdc6lhAVRtOVoC3F9QIjTd9cp8lrH3+nJ052d/8yenl50atXLxYtWkS3bt0AWLZsGV5eXnTr1g2DwUBoaGjx4999911WrlzJ6tWref755ystvxCieloZnUhaTgE+rvbkFRiJSUij72fbmTmsFfc38tU6nhD3TEaArMioUaNYsWJF8eaxCxcuZPjw4RgMBrKzs/m///s/mjZtioeHBy4uLpw8eVJGgISwQoqiMH/XeQCe7lqXNZO60KKWO2k5BYyfv5+PN5zGaJJNBIR5kxGgCuBoa+D4Oz01e+3S6t+/PyaTiTVr1tC2bVu2b9/Oxx9/DMArr7zCunXr+O9//0v9+vVxdHTk0UcfJT8/v7KiCyGqqZ1nrnD6UhbOdgaGtg3EzcGWZc9E8O9fjrNwbzyf/f4HB+Ou8enwVtRwsdc6rhDlIgVQBdDpdKW6DKU1R0dHHn74YRYuXMiZM2do2LAhYWFhAGzfvp1x48YxePBgALKysoiNjdUwrRBCK3N3qqM/Q8LV4gfAwdbAfwa3IDzYk3/+eJQdZ1Lp+9kOvhjVmrAgLy3jClEucgnMyowaNYo1a9Ywd+5cRo8eXXy8fv36/Pjjj8TExHDo0CFGjhx504wxIYTlO5+azaaTKeh0MLZj8E2fH9y6Nque70RdH2eSM3IZ9tUevt1+rtQTMoSoLqQAsjIPPvggXl5enDp1ipEjRxYf/+STT/D09KRjx47079+fnj170qZNGw2TCiG0sGBXLAAPNPIlxNv5lo9p6OfK6uc7069lAIUmhXfXnOC5hQfJzJUNj4X50ClStt8kIyMDd3d30tPTcXNzK/G53Nxczp8/T0hICA4ODholNC/ydyaEecjILSBi+u9k5xv54fH2dG7gfcfHK4rCd7vjeHfNcQqMCiHezswe1YYmAW53fJ4QleVOP7//TkaAhBCayy2aZr1wbxyR++IxyQwjTSzdn0B2vpGGfi50ql/jro/X6XSM7RjM0qcjqOnuwPnUbAZ9sZNlBxKqIK0Q96b6d+4KISxKek4Bx5LSOX4xg2MXMzh2MZ2zl7NLTKu2Neh5NKy2himtj9GksGB3LADjOoag0+lK/dzWdTxZM6kLU5bEsPX0ZV5ZfpiouGu8NaAZDmWYqSpEVZICSAhRKRRFISUzj6OJ6cWFzrGLGVy4duvVhGs42+HtYs+pS5l8ufUsD7euhV5f+h/C4t5sPHGJhKvX8XCyZXDrWmV+vqezHfPGteXzzWf4ZONpFu9P4PCFdOaMbkNQjVv3EgmhJSmAhBD3zGRSiLuaU1zkHLuYwfGL6aRm3XodqdqejjSr6Uazmu7Ff/q52ZOVV0in9zZxJiWL9ccv0au5fxW/E+s1r2jq+4h2dXC0K9+ojV6vY1K3BrSp48mkxdEcT8qg36wd/HdIKD2byddSVC9SAAkhyiS/0MQfKZlFRY46snMiKZOsvMKbHqvXQX1fl+JCp2lNN5oFuOPuZHvLc7s62PJYRDCfbz7D7C1n6NnMr0yXYkT5HL+YwZ5zVzHodYzpEHTP5+vcwJs1kzrz/KJoouKu8fT3UTzVtS6v9GyErUFaT0X1IAWQEOK2svMKOZGUUeIS1h+Xssg33rxGlL2NnsYBbkUjOuqoTmN/1zL3gIzvFMy3O85x+EI6O89cuetMJHHvboz+9G7uT00Pxwo5Z4C7I4uf6sD7v57k2x3n+XrbOWLi05g1sjV+bjIbVGhPCiAhBABXsvKKL18du6g2KZ+/ks2tFspwdbApcQmreS136no7Y1MBv93XcLFneNs6zN8Vyxebz0gBVMmuZOWx6tBFAMZ3CqnQc9sa9LzerylhQZ68svww+2Kv0vez7Xw2ojUd68nXVWhLCiAhrIyiKCSmXVcLncQ/e3aSM3Jv+Xg/N/u/9OqoRU9tT8dKvTT1VNe6/LAnjt3nrnAw/hpt6nhW2mtZu0V748kvNBFa2502dTwq5TV6twigcYAbz/4QxcnkTEZ/u5eXejTi2fvqSaO70IwUQEJYMKNJ4dzlrBKXsI5dzCD9+q1X7A3xdlb7dP4yuuOtwWaXNT0cGdy6FsuiLjB781m+HRte5RmsQX6hie/2xAEwoXPZpr6XVYi3Myuf68Qbq46yPOoCH647RVTcNT4eGoqHk12lva4QtyMFkBAWIrfAyKnkzBLFzsnkDHILbu7XsdHraODnWmJUp0mAK64Ot25O1sIz99dj+cELbDxxiVPJmTTyd9U6ksVZeySJy5l5+Lra07t5QKW/nqOdgf8OCaVdsBdvrDrKppMp9P1sB3NGt6FlbY9Kf30h/koKICty//3306pVK2bOnFkh5xs3bhxpaWn89NNPFXI+UT4mk8JrPx1h6YELJRYTvMHJzkCTvzUnN/Bzwd6mei9QV8/Hhd7N/Vl7JJk5W84wc3hrrSNZFEVRind9H9MhCDubqpudNbRtIM1qufHcwoPEXcnh0Tm7eaN/U0a3ryOz/kSVkQJICDP3zfZzRO5Ttx7wcrb7c7p50SWs4BrOGMy0z+K5++uz9kgyPx9O4qUejQj0ctI6ksU4GH+NwxfSsbPRM7J9nSp//WY13Vn9fGdeWXaI9ccv8cZPR4mKvcp/BrfA2V5+NInKJwsyWIlx48axdetWPv30U3Q6HTqdjtjYWI4fP06fPn1wcXHBz8+PMWPGkJqaWvy85cuX06JFCxwdHalRowYPPfQQ2dnZvPXWWyxYsIBVq1YVn2/Lli3avUErFRV3jQ/WnQLg34OaE/X6Q3z/eHum9W7CgNCa1PNxMdviB6B5LXe6NvTBaFL4attZreNYlLk7YwEY1KomNTTo8wJwd7TlqzFhvNanCQa9jp9iLjLoi52cScnUJI+wLlIAVQRFgfxsbW63mqN8C59++ikRERE8+eSTJCUlkZSUhK2tLffddx+tWrXiwIED/Pbbb1y6dImhQ4cCkJSUxIgRI5gwYQInTpxgy5YtPPzwwyiKwssvv8zQoUPp1atX8fk6duxYmX/L4m/ScvKZFBmN0aTQP7SmxV4+eO7+egAsPXCBlMxbz1QTZXMx7Tq/HU0GKn7qe1npdDqe7FqXyCc74Otqzx8pWQz4fCeri6bmC1FZZJyxIhTkwPSa2rz2Py+C3d332XF3d8fOzg4nJyf8/dUl6d98803atGnD9OnTix83d+5cAgMDOX36NFlZWRQWFvLwww8TFKSuDtuiRYvixzo6OpKXl1d8PlF1FEXh5WWHSUy7TnANJ6YPbm6RxQ9A+xAvwoI8iYq7xv92nGda7yZaRzJ73+2Ow2hS6FDXiyYBblrHAaBdiBdrJnVhUmQ0u89dYVJkNAdir/Ja3ybVvl9NmCcZAbJiUVFRbN68GRcXl+Jb48aNATh79iyhoaF069aNFi1aMGTIEL755huuXbumcWoB6uWLjScuYWfQ8/nINtVq9lZF0+l0xaNAP+yOIz3n1lP4RelczzcSuS8egAkaj/78nY+rPT880Z7nH6gPqIXa0K/2cOFajsbJhCWSEaCKYOukjsRo9drlZDKZ6N+/P++///5NnwsICMBgMLBhwwZ27drF+vXrmTVrFq+99hp79+4lJKR6feO0JocS0njv1xMAvNGvCc1ruWucqPI92NiXxv6unEzO5LvdsbzQrYHWkczWyuhE0q8XEOjlSLcmflrHuYlBr+Plno0IC/JkypIYDiWk0W/WDj4Z1ooHGvlqHU9YEBkBqgg6nXoZSotbGS572NnZYTQaiz9u06YNx44dIzg4mPr165e4OTs7F701HZ06deLtt98mOjoaOzs7Vq5cecvzicqXfr2AiYsOUmBU6NPCn9EVsHGlOdDpdDxbNAo0d+d5cvJv3nhV3J2iKMX7fo2NCK7WDfIPNPZlzaTOhNZ2Jy2ngPHz9vPR+lO3XOpBiPKQAsiKBAcHs3fvXmJjY0lNTWXixIlcvXqVESNGsG/fPs6dO8f69euZMGECRqORvXv3Mn36dA4cOEB8fDw//vgjly9fpkmTJsXnO3z4MKdOnSI1NZWCArk0UZkUReEfyw9z4dp1Ar0cee+Rlhbb93MrfVsEEFTDiWs5BSwumvYvymbHmVT+SMnC2c7A0LaBWse5q9qeTix9JoLHItRCf9amMzw2dy+pWXkaJxOWQAogK/Lyyy9jMBho2rQpPj4+5Ofns3PnToxGIz179qR58+ZMnjwZd3d39Ho9bm5ubNu2jT59+tCwYUNef/11PvroI3r37g3Ak08+SaNGjQgPD8fHx4edO3dq/A4t23e74/jtWDK2Bh1fjGyDmwX3/dyKjUHP013VUaBvtp8jv/DmFa7Fnc0rmvo+JDzQbP792NsYeGdgcz4d3gonOwM7z1yh72fbORB7VetoVUpRFBKu5rDt9GWy82QEtCLoFKWU86itSEZGBu7u7qSnp+PmVnKGRG5uLufPnyckJAQHBweNEpoX+Tu7d0cT03l49i7yjSbe7NeUCZ2tswcrr9BIl/c3k5KZxwePtDSLUYzq4tzlLB78aCs6HWx66X5CvO8+e7S6+eNSJs8uPMiZlCwMeh3Tejfm8Urew0wLhUYT51Kz1S1tEtX9+44n/bmHX6tADyKf7ICjncyO+7s7/fz+O2mCFqKay8xV+37yjSZ6NPVjfKdgrSNpxt7GwJNd6vKftSeYs/Usj4TVrtZ9LNXJgl2xADzYyNcsix+ABn6urJrYiWk/HmH1oYu8u+YE+2Ov8uGQULMZ0fq73AIjJ5MzS2xWfDIpg7xbjHDaGnTodTpiEtKYsiSa2aPC5N//PZACSIhqTFEUpv14hLgrOdTycOTDR0Mt7rfdshrRvg6fbz7D+dRsfjuaTN+Wlb+Jp7lLv17AsqgLgPYLH94rZ3sbPh3eirYhXvz75+OsO3aJU8k7mD0qjKY1q8eaRreTfr2A43/ZrPjYxXTOXs6+7R5+TQP+3L+vaU03Gvi5cCghndHf7mXdsUv8Z80J3uzfVIN3YhmkABKiGlu0L55fDidho9cxa2Rr3J3M87fciuRib8O4jsF8+vsfzN5yhj4t/K2+KLybZQcSyMk30tDPhU71a2gd557pdDrGdAiiZS13nlt4kNgrOQyevZN/D2rO0HDtL4sqikJKZl6JS1jHktJJuHr9lo+/sYffjf37buzhp7/F6E67EC/+OzSUSZHRzN15nkAvR7MvarUiBZAQ1dTxixm8/fNxAP6vVyPa1PHUOFH1Ma5jMN9sP8exixlsPX2Z+2V9mNsymhTmF13+Gt/JsvplQgM9+OWFzkxdGsPmU5f5v+WHORB7lXcGNsfBtmr6Y0wmhbirOSUuYR2/mE5qVv4tH1/Lw7FksVPLDX83hzJ9XQaE1uTCtRw++O0U7/xynFoejvRoJivyl5UUQOUkveOlJ39XZZeVV8jziw6SX2jiwca+PNG5rtaRqhVPZztGtqvDtzvOM3vLWSmA7mDjiUtcuHYdDydbBrWqpXWcCufpbMf/xrZlztazfLT+FEsPXOBIYgZzRrUhuIJ7nfILTfyRkllU5KiXsE4kZZJ1i1lZeh3U83EpUew0remGh5NdhWR59r56JFy9TuS+eCYtjmbxUxG0CvSokHNbCymAysjWVr0EkZOTg6Ojo8ZpzENOjrqM/Y2/O3FniqLw+sojnEvNJsDdgY+GhN5yKNzaPdGlLgt2x7Lv/FUOxF4lPNhL60jV0twd6sKHI9rVsdhZQ3q9jokP1Kd1oAeTFkdzIimD/rN28OGQUHo1L9/ISHZeISeTMzia+GfPzh+Xssg33tycbGejp4m/K03/cgmrsb9bpf5963Q6/j2wGRfTrrP19GWeWLCflc91ItCr/LsDWBuZBn8Ld5tGl5SURFpaGr6+vjg5OVnUkHJFUhSFnJwcUlJS8PDwICBAmlVLY8n+eP6x4ggGvY4lT3WQH+x3MO3Hw0TuS+DBxr7MHddW6zjVzrGL6fT9bAcGvY4d/3iAAHfL/6UtOT2XFyIPsj9W3bfwyS4h/F+vxtgabr/s3dXs/BKXsI5dTOd8aja3+uno6mDzt34dd+r5OGNzh/NXpqy8QoZ+uZvjSRnU83Hmx2c7WXWvoEyDr2Q3dj9PSUnROIl58PDwkB3jS+lUcib/Wn0MgJd6NJTi5y6e7lqPJfsT2HQyheMXM6r9LKCqNr9o4cPezf2tovgB8Hd3YNGTHfhw3Sm+3naOb7afJzo+jc9HtsHPzZ7EtOslenWOXcwgKT33lufydbWnea0/R3Wa1XSntqdjtfql18Xehrnj2jJ49k7OXs7mqe8P8N3j7bC3sczRvookI0C3UNoK0mg0yvYPd2Fra4vBIP8RSyMnv5ABn+/kTEoW9zX0Yd64tnLpqxReiIzm50MX6R9ak1kjWmsdR3VsJZzfDg++Dk7aFLGpWXl0nLGJfKOJFc92JCzI+profzuazCvLDpGZV4ibgw16vY60nFt/zw6u4VQ83fxGsePjal/FicvvRFIGQ77cTVZeIQNb1eSToa2s8vuHjABVEYPBID/cRYV5c9UxzqRk4edmz8dDpe+ntJ69rx4/H7rImsMXeal7wwpvfC2Tglz47VWImqd+bLCF3u9rEmXR3njyjSZCAz1oU8dDkwxa69Xcn8b+rjy78CAnkjIAsNHraODnWmJUp0mAK65mupDiDU0C3Jgzug3j5+1nVcxFAj2deLlnI61jVWtSAAlRDayIusDyqAvodfDp8NbUcDGf3zy11rSmGw829mXTyRS+2naWGQ+31CbItVhYOhaSYv48FrUAur4Czt5VGiW/0MT3e+IAmNApuFpdsqlqwd7OrHyuI3vPX6WGsx0N/Fws9vJQlwY+TH+4Bf+3/DCfbz5DbU9Hhrero3Wsaks2QxVCY2dSMnn9p6MATHmoIR3qmv9CdVXtufvVTVJXRCWSfJt+jkp16lf4qqta/Dh6wegVENAKCq/DnjlVHmfNkYtczszD19We3s1l8oGDrYH7GvrQvJa7xRY/NwwND2TSg/UBeO2no2w9fVnjRNWXFEBCaOh6vpGJC6O5XmCkU/0aTHygvtaRzFJ4sBftgr3IN5r4dvu5qnthYyFsfAsih0NuOtRuC89sh/oPQZep6mP2fQO5GVUWSVGU4l3fH4sIws5Gvs1bmxe7N2Rw61oYTQoTFx7k+MWq+/dnTuR/hhAaevvnY5y6lIm3iz0zh7WWjQ3vwXMPqKNAi/bFcy371qvwVqjMS/D9INjxifpx+2dh3Fpwr61+3Lg/eDeEvHQ48L/Kz1PkYPw1Dl9Ix85Gzwi5/GGVdDod7z/Skg51vcjKK2TC/P0kpd96Gw5rJgWQEBpZFZPI4v0J6HTw6fBWZjXjpDq6r6EPzWq6kZNvLN76odLE7oSvukDsdrBzgUfnQe/3wOYvq/zq9dBpinp/92woqJofQHN3xAIwqFVN6SWzYnY2er4aHU59XxeSM3IZP28/mbkya/mvpAASQgPnLmfxzx+PAPDCgw3oVL9qm2QtkU6n47n71UuI83fF3nJ7gnumKLBjJizoD1mXwKcJPLUFmj9868e3HArugZCdAjELKz7P31xMu85vx5IB89/1Xdw7dydb5o1ri7eLPSeTM5m4KJqCW6xkba2kABKiiuUWGJm4KJrsfCMd6noxuVsDrSNZjF7N/anr7Uz69QIi98ZX7Mmvp8HikbDxX6AYoeVwePJ38L7D189gCx1fUO/v/FTtGapE3+2Ow2hSiKhbgyYBsiikgEAvJ+aOC8fR1sC205d546ejsj9jESmAhKhi7645zomkDGo42/HpcOn7qUgGvY5n7lN7gb7Zfo68QmPFnPhijDrL69RaMNhBv5kw+EuwK8WaQ63HgJM3pMXD0RUVk+cWcvILidynFn3jOwVX2usI89OytgezRrRGr4PF+xOYveWs1pGqBSmAhKhCvxy+yA971B9SHw9rhZ+bg8aJLM+g1rUIcHcgJTOPFVGJ93YyRYGo+fC/HpAWBx5B8Ph6CB8PpV1bx84JOjyr3t/xMZgq5xLEyuhE0q8XUMfLiW5N/CrlNYT5eqipH//q3wyAD9edYlXMPf7fsABSAAlRReKuZPPqCrXv57n763FfQx+NE1kmOxs9T3apC8BX285SWN6eh/wc+OlZ+HkyGPOgYW94eivULMd2G22fAHs3uHwSTv9avjx38Nep72M7BsuoorilsR2DeaKz2hv2yrLD7D13ReNE2pICSIgqkFdoZOKig2TlFdI22JOp3RtqHcmiDW8XiKeTLXFXclhzJKnsJ0g9A992g0ORoNPDQ2/D8EXgWM79tBw9oO3j6v3tH3HLbcbvwfY/UjmTkoWznYEh4bUr9NzCsvyzTxN6N/cn32jiqe+jOJOSpXUkzUgBJEQVmLH2JEcTM/B0suWzEa2xMch/vcrkZGdTPAtqzpazZWv6PLYSvr4fUo6Dsy+M/Rk6T1Gntd+LDs+BjQMkRsH5bfd2rr+Zt/M8AEPCA3Ez8z2tROXS63V8MqwVret4kH69gHHz9nE5M0/rWJqQ78JCVLLfjiYXr0vz8dBWBLg7ahvISoyNCMbZzsDJ5Ew2n0q5+xMK8+HXV2HZOMjPhKDO6qrOwZ0rJpCLr9oQDeooUAU5ezmLzacuo9PBuI7BFXZeYbkcbA18+1g4QTWcuHDtOk98d4Dr+RU0YcCMSAEkRCVKuJrD/y0/BMDTXevyQGNfjRNZD3cnW0Z3CALgi813GQVKvwDz+8Deon27Or8Ij60CV/+KDdVpEuht4PxWuBBVIadcUFRcP9jIl2DvUsxKEwKo4WLPvHFt8XCy5VBCGpMWR2M0Wdf0eCmAhKgk+YUmno+MJiO3kNZ1PHi5ZyOtI1mdxzuHYGejJyruGvvOX731g878rk5xv7AfHNxhxGJ46C0w2FR8II860GKIen/Hx/d8uvTrBSyPugDAhM6y8KEom7o+LnzzWDh2Nno2HL/Eu2uOax2pSkkBJEQl+eC3kxxKSMPd0ZZZI1pjK30/Vc7XzYEhYWpT8Bd/X/vEZIQt78EPj0DOFQgIhae2QqPelRuq84uADk7+Aikn7+lUS/cnkJNvpJGfKx3r1aiYfMKqtA324qMhoQDM2xnL3B3nNU5UdeQ7shCVYOPxS3xb9I3kw0dbUtvTSeNE1uvprvUw6HVsO32Zo4np6sHsK7DwUdgyA1AgbDxMWA9eVTCK4tMIGvdV7++cWe7TGE0KC3bHAjCuUzC60q5LJMTf9A+tyau9GwPw7zXH+e1ossaJqoYUQEJUsMS067y0TO37mdAphB7NKriPRJRJnRpO9G8ZAMDsLWcgYb+6kenZTWDjCIO/gv4zwbYKF6XsMlX98/BSuBZXrlNsOH6JC9eu4+lky+DWtSownLBGT3ety8j2dVAUmLIkmuj4a1pHqnRSAAlRgQqMJl5YdJD06wWE1nYv/q1KaOvZ++sDCn4n5qPM6wUZiVCjvrqXV+jwqg9UKwzq3q/uKbZrVrlOMbdo6vuIdnVwsDVUYDhhjXQ6He8MaMYDjXzILTDxxIIDxF/J0TpWpZICSIgK9N/1pzgYn4argw2fj2yDnY38F6sOGnkoLKvxNf+y+Q6dqRCaDoInN4NfM+1CdS4aBYr+HrJKMU3/L45dTGff+asY9DrGRARVQjhhjWwMej4f2YZmNd24kp3PuPn7SMvJ1zpWpZHvzkJUkM2nUvhq6zlA7fsJ9JK+n2rh0jH45gHaZm+lQDHwTuFYLnafAw4a75Ye0hVqhUNhLuyZXaan3tj2ok+LAFlXSlQoZ3sb5o5rS013B85dzuap76IqblPhakYKICEqQHJ6Li8tVft+xkYE0at5gMaJBACHFsM33eDKGXCrxb99/svcwp58Ux1muuh0f/YC7fsWrqeV6mmXM/NYHXMRkF3fReXwc3Ng3vh2uNrbsC/2Kq8sO4zJAtcIkgJIiHtUaDQxKTKaq9n5NKvpxrQ+TbSOJApy1U1MVz4Nhdeh3oPw9Ha69+wPQOS+eK5kVYPl/xv2Bp8m6srT+78t1VMW7Y0n32giNNCDNnXKuTeZEHfRyN+VOaPDsNHrWH3oIv9df0rrSBVOCiAh7tHMjX+wL/YqLvY2fDGyjTSkau3qefhfd4iaD+jg/mkwajk416BzfW9a1nYnt8BUvD2JpvT6onWBUC+D5d+56TSv0MgPe9VZYxNk9EdUss4NvJnxcAsAZm85S+S+eI0TVSzNC6DZs2cTEhKCg4MDYWFhbN++/baP/fHHH+nevTs+Pj64ubkRERHBunXrSjxm/vz56HS6m265ubmV/VaEFdr+x2W+2HIGgBkPt5CtCLR2ci18fR8kHwZHLxi9Au5/FfRqUarT6Xju/noAzN8VS2ZugZZpVc0fUVeIzrmiNkTfwZrDSVzOzMPPzZ4+LeQyq6h8Q8IDmdStAQCv/3SULaXZV89MaFoALVmyhClTpvDaa68RHR1Nly5d6N27N/Hxt64yt23bRvfu3Vm7di1RUVE88MAD9O/fn+jo6BKPc3NzIykpqcTNwaEK1/gQViElI5cpi2NQFBjZvg79Q2tqHcl6GQthw79g8QjITYfabdWNTOt3u+mhPZr6U8/HmczcQhburQa/0RpsoNNk9f7Oz9RNWW9BUZTi5ucxHYJkZXFRZV58qAEPt66F0aQwceFBjl1M1zpShdD0f9DHH3/M448/zhNPPEGTJk2YOXMmgYGBzJkz55aPnzlzJv/3f/9H27ZtadCgAdOnT6dBgwb8/PPPJR6n0+nw9/cvcROiIhlNCpMXx3AlO5/G/q682a+p1pGsV2YyfDfgz1WV2z8L49aCe+1bPlyv1xWtCwTfbj9PbkE1mOHSajQ4+0LGBTiy7JYPiYq7xpHEdOxt9IxoV6eKAwprptPpeO+RlkTUrUF2vpEJ8/eTlH5d61j3TLMCKD8/n6ioKHr06FHieI8ePdi1a1epzmEymcjMzMTLy6vE8aysLIKCgqhduzb9+vW7aYTo7/Ly8sjIyChxE+JOZm36g93nruBkZ+CLUdL3o5nz2+HLLhC3E+xcYMh86P0e2Njd8WkDW9WklocjqVl5LCvaTFRTtg4QMVG9v+MTdZ+yv7mx8OGgVrWo4WJflemEwM5Gz5djwmjg68KljDzGz9tfPS4h3wPNCqDU1FSMRiN+fn4ljvv5+ZGcXLp9SD766COys7MZOnRo8bHGjRszf/58Vq9eTWRkJA4ODnTq1Ik//vjjtueZMWMG7u7uxbfAwMDyvSlhFXadTeXT39V/T9MHt6Cej4vGiayQyQTbP1ZHfrJTwLcpPLUFmg0u1dNtDXqe6loXgK+2nqXQaKrEsKUUPkHdjf7KH+pGqX+RmHaddccuATC+c7AG4YQAd0db5o1vi4+rPSeTM3lu4UEKqsP/nXLS/CLy3zfwUxSlVJv6RUZG8tZbb7FkyRJ8fX2Lj3fo0IHRo0cTGhpKly5dWLp0KQ0bNmTWrNsvNz9t2jTS09OLbwkJCeV/Q8KiXc7MY3JR38/Q8NoMkj2Yqt71a2qvz+9vg2KC0BHwxO/g3aBMpxnWNhBvFzsuXLvOz4cvVlLYMnBwg3ZPqfe3fwzKn+uufLc7FqNJIaJuDRr7a7yAo7BqtT2d+N/YcBxtDWz/I5XXVx5FUcxzjSDNCiBvb28MBsNNoz0pKSk3jQr93ZIlS3j88cdZunQpDz300B0fq9fradu27R1HgOzt7XFzcytxE+LvTCaFqUtjuJyZR0M/F94e0FzrSNYn8SB81RVO/wYGe+j/KQyaA3ZlX3XbwdbA+E7q7u+zN5+tHgu9tX8WbJ0gKUbdrBXIyS8ksqhZe0LnKtitXoi7aFnbg89HtkavgyUHEvhi8xmtI5WLZgWQnZ0dYWFhbNiwocTxDRs20LFjx9s+LzIyknHjxrFo0SL69u1719dRFIWYmBgCAmTKqLg3s7ecYfsfqTjaGvhiZBsc7aTvp8ooChyYC3N7Qlo8eATB4+shbJy6onI5jYkIwtXehj9Ssth44lLF5S0v5xrQZqx6f8cnAPx4MJGM3ELqeDnxYGPfOzxZiKrTrYkfbw9Q99L77/rT/BSdqHGistP0EtjUqVP59ttvmTt3LidOnODFF18kPj6eZ555BlAvTT322GPFj4+MjOSxxx7jo48+okOHDiQnJ5OcnEx6+p9T8t5++23WrVvHuXPniImJ4fHHHycmJqb4nEKUx95zV/h4w2kA3hnYjAZ+rhonsiL52eqKzr+8CMZ8aNQHnt4KNVvd86ndHGyLNxP9YsvZ6jGU3/F50NtC7HZMcXuZV9T8PLZjMAZ9+Ys9ISramIhgnuyijkq+svwQe85d0ThR2WhaAA0bNoyZM2fyzjvv0KpVK7Zt28batWsJClK/ISUlJZVYE+irr76isLCQiRMnEhAQUHybPHly8WPS0tJ46qmnaNKkCT169CAxMZFt27bRrl27Kn9/wjJcycpj0uJoTAo83KYWQ8KlSb7KXD6t7uV1eAnoDPDQ2zB8EThW3BYQEzqHYG+j51BCGrvPVoNv4O61IXQYAFfXv8fZy9m42NswNPzW0/qF0NK03k3o08KfAqPCU98d4ExKptaRSk2nVItfeaqXjIwM3N3dSU9Pl34gK2cyKYyfv5+tpy9Tz8eZ1c93xtneRutY1uHoClg9CfKzwMUPHp0LwZ0r5aX+teooC3bH0bm+Nz880b5SXqNMUv+Az9sCCj3z3iMioitvFV1uEKK6yS0wMvKbPRyMT6O2pyMrn+uEj6s2SzWU5ee35rPAhKjOvt5+jq2nL2Nvo+eLUW2k+KkKhfmw9v9g+QS1+AnuAk9vr7TiB+DJrnWx0evYcSaVQwlplfY6pebdgKx6ao/jszarGdcxWNs8QtyBg62Bbx4LJ6iGExeuXefxBfvJyS/UOtZdSQEkxG1ExV3lw3XqDshvDWgm04+rQvoFmNcb9n2lftz5RRjzE7jeeWbovart6cTAVuqSBrO3VI8ZLd8ZHgFggGEPwfpq0KAtxB3UcLFn/vh2eDrZcvhCOpMiYzBWh5mVdyAFkBC3cC07nxcWRWM0KQwIrcnwttL3U+kun4ZvH4LEA+qCgCMWw0NvqXtlVYFn76+LTgfrjl3ij0va9jGkXy/g85NObDGGosek7hEmRDUX4u3MN4+FY2ejZ+OJS/z7l+PVY2LBbUgBJMTfKIrCK8sPcTE9lxBvZ6Y/3KJUi3OKe5B8RB35yUwCnybw9DZo1LtKI9T3daVnU3XfwDlbz1bpa//d0v0J5OQb+dltuHogZqG655kQ1Vx4sBefDG0FwPxdscwt2sC3OpICSIi/+d+O82w8kYKdjZ7PR7bGRfp+KteFKJjfD3JSISAUxq0Bz2BNojz3QD0AVsVcJOFqjiYZCo0m5u+KBaBt174Q2EGd/r/7c03yCFFWfVsGMK13YwDeXXOc344maZzo1qQAEuIvouOv8d6vJwF4o19TmtV01ziRhYvbBd8NhNw0qN0OHlutLgaokZa1PejSwBujSeGb7ec0ybDxxCUS067j6WTLoDa1octU9RMH5kHOVU0yCVFWT3Wty+gOdVAUmLw4hoPx17SOdBMpgIQokp5TwPOLoik0KfRtEcDo9nW0jmTZzm6C7x+G/Ex1pteYleDooXUqnr1fHQVasj+By5l5Vf76Ny4ZjGhXBwdbAzToAX7N1Rlx+76p8jxClIdOp+Ot/s14sLEveYUmnlxwgLgr2VrHKkEKICFQ+37+b8UhEtOuU8fLiRmPSN9PpTr1KywaBoXXoX53GLUM7F20TgVARN0atAr0IK/QxNyiVZirytHEdPadv4qNXle8QjU6nTobDmDvHMjLqtJMQpSXjUHPrBGtaV7LjSvZ+Yyft59r2flaxyomBZAQwIJdsaw7dglbg44vRrbBzcFW60iW6+iPsGS02tfSuB8MXwi2jlqnKqbT6Zj4QH0AftgdR/r1gip77XlFoz+9WwQQ4P6Xv5Nmg8GrLly/BgcXVFkeIe6Vs70Nc8e2paa7A+dSs3nq+wPkFhi1jgVIASQERy6kM32t2vfzzz5NaFFb+n4qTcwiWPE4mAqhxRAYsgBstFkx9k66NfaloZ8LmXmF/LAnrkpe83JmHj8fugjA+E7BJT+pN0Cnoi1/dn0OhVV/aU6I8vJ1c2De+Ha42tuwP/Yaryw/jKkarBEkBZCwahm5BUxcdJB8o4mezfxkxd3KtP9b+OlZUEzQ5jEY/FWVrfFTVnq9jufuV0eB5u44z/X8yv+NdeHeOPKNJloFetCmzi32OgsdAa4BkHkRDi2u9DxCVKRG/q58OSYMG72Onw9d5MP1p7SOJAWQsG6vrTxK/NUcans68sEjodL3U1l2zYI1L6n32z8D/T5VRzWqsX4tAwj0cuRKdj5L9sff/Qn3IK/QyA971Ne4afTnBht7iHhevb9zJpiqx2UEIUqrU31v3nukJQBztpxl0d7K/X91N1IACat1KjmTnw9dxKDXMWtEa9ydpO+nwikKbP0A1r+uftx5KvR6D/TV/1uPjUHP013VGWFfbztHfqGp0l5rzeEkUrPy8HOzp0+LgNs/MGwcOHrC1XNw/KdKyyNEZXk0rDaTuzUA4NPfT2u6Z1j1/y4kRCVZtFft7ejexI/Wt7rkIO6NosDGt2Dzf9SPH3wdHvqXOqvJTDwaVhsfV3supueyKiaxUl5DUZTi2WaPRQRja7jDt2V7F3UEDWD7J+rfsRBmZspDDZj0YH2WP9MRJzvtLoNLASSs0vV8Iz9Gqz/QRsp6PxXPZIJf/6FeqgHoOR26vqJppPJwsDXwROcQQN0eozI2dzwQd42jiRnY2+gZ0a4U/xbbPQW2znDpCJzZWOF5hKhsOp2OqT0aEejlpGkOKYCEVfrl8EUycwup4+VE5/reWsexLCYj/PxC0Y7uOuj3CURM1DpVuY3qEISbgw3nLmez/ljF78c1r2j0Z1CrWng52939CU5eED5evb/9owrPI4S1kAJIWKVF+9Tmu+HtAtHrzeeSTLVnLIAfn4LoH0Cnh8FfQvgErVPdExd7G8YWzQ6cveVshe5ufeFaDr8dVYuq8Z2DS//EiOfBYAfxu9XtRIQQZSYFkLA6J5IyiI5Pw0av49Gw2lrHsRyFebBsHBxdDnobeHQuhA7XOlWFGN8pBEdbA0cS09lxJrXCzvv97jhMCnSsV4PG/m6lf6JbALQaqd7f/nGF5RHCmkgBJKzOjamXPZr54evqoHEaC5GfA4tHwslfwGAPwxaqqxdbCC9nO4a3CwTgi81nKuScOfmFRO67MfU9pOwn6DRZHWU7swGSDldIJiGsiRRAwqrk5Bfy043m53ZBGqexEHmZsGio2pBr6wQjl0CjXlqnqnBPdqmLrUHHnnNXiYq7952tVxxMJCO3kKAaTjzY2LfsJ/CqC80eVu/vkFEgIcpKCiBhVX45lERmnvpDp2O9GlrHMX/X0+D7wRC7HexcYfSPUO8BrVNVipoejgxuXQuAOVvubRTIZFKYX9T8PDYiGEN5+9BubJJ67CdIrZiRKSGshRRAwqosLLrkMKJdHWl+vlfZV2BBf7iwHxw8YOwqCIrQOlWleua+euh0sPFECieTM8p9nu1nUjl7ORsXexuGhN9DH5p/c2jYC1D+XHJACFEqUgAJq3HsYjqHEtKwNUjz8z3LTIb5fSD5MDj7wLg1UCtM61SVrq6PC32aqys1z9lyttznmbtDHf0ZEl4bV4d7XIG881T1z0OLIb1yFmsUwhJJASSsxp/Nz/54u1S/HcjNRloCzOsNl0+qm3OOW6uORFiJZ+9Xt8f4+dBF4q/klPn5Z1Ky2Hr6MjodFbP5bp32ENQZTAWw+/N7P58QVkIKIGEVsvMKWRVzEYBRpVltV9zalbNq8XP1HHjUgfG/gk9DrVNVqea13LmvoQ8mBb7cVvZRoAW7YgHo1tiXoBrOFROqS1EvUNR89dKkEOKupAASVuHnQxfJyiskuIYTEdL8XD4pJ2FeH0hPgBr11eLHqxzTty3Ac0WjQMsPXCAlI7fUz0vPKWB51AUAJpRn6vvt1OsG/i2hIKdoBW4hxN1IASSswqK/ND/rzGgzzmoj6bDa85OVDL5N1eLH3Xr7qNqFeBEe5Em+0cS3Rf08pbHkQDzXC4w08nOt2EJcp4MuL6n3936pLk0ghLgjKYCExTuamM7hC+nYGfTS/FweFw7Agn6QcwUCWqkNzy7lWLfGguh0Op57QB0FWrgnjvScgrs+p9BoYsGuOADGdwqu+EK8SX91ZC43HQ7Mq9hzC2GBpAASFu/G6E/P5v7UkObnsondCd8NVH+oBraHsavVzTgFDzTypbG/K9n5Rhbsjr3r4zccv0Ri2nU8nWwZVLSeUIXSG/5cF2j351BQ+ktzQlgjKYCERcvKK2RV8crP0vxcJmd+hx8egfwsCOmqLnLo4K51qmpDHQWqD6g7uufkF97x8fN2xgIwsn0dHGwNlROqxVBwqwVZl+DQosp5DSEshBRAwqKtjrlIdr6Rut7OdKgrIxeldnINRA6HwuvQoAeMXAr2Llqnqnb6tggguIYT13IKiNyXcNvHHU1MZ1/sVWz0OsZ0CK68QDZ20HGSen/HTDDeuSgTwppJASQs2qJ9as+FND+XwdEVsGQMGPOhyQB1Y1NbR61TVUsGvY6n71N7gb7Zdo68QuMtHze3aNuLPi0C8Hev5A142zwGTjUgLQ6Orazc1xLCjEkBJCzWkQvpHE3MwM6g5xFpfi6d6IWw4glQjNByGDw6Tx1VELf1cJta+LnZk5yRW7zR7l9dzszjl0NJgNr8XOnsnKDDs+r9HR+DyVT5rymEGZICSFisG6M/vVv44+UsP8Tvat83sOo5UEzQZiwM+hIMNlqnqvbsbQw82aUuAF9uPYfRpJT4/MK9ceQbTbQK9KB1Hc+qCdX2SXVz2pTj8Me6qnlNIcyMFEDCImXmFhSv/CzNz6Ww81NY+7J6v/2z0P9T0Mu3h9Ia0a4OHk62nE/N5tejScXH8wqN/LBHLcQndK7CRSMdPaDt4+r97R+Botzx4UJYI/kOJyzSqpiL5OQbqefjTLsQaX6+LUWBLe/BhjfVj7u8DL1mqAvriVJztrcp3tfri81nUYoKjl8OJZGalY+/mwO9m/tXbagOz4HBHi7sh9gdVfvaQpgBKYCExVEUpXjjU2l+vgNFUQufLTPUjx98A7q9IcVPOY3rGIyTnYETSRlsOX0ZRVGKm5/HRARha6jib7euftBmjHp/x8dV+9pCmAEpgITFOXwhneNJGdjZyMrPt2UywdpXYNdn6sc9Z0DXl7XNZOY8nOwY1V693Dp78xn2x17j2MUM7G30jNDqMmzHSaAzwNlNcDFamwxCVFNSAAmLc2P0p2+LADycpPn5JiYjrH4B9n8D6KDfTIh4TutUFuGJLnWxM+jZH3uNf648AsDg1rW0a8L3DIIWj6r3t8sokBB/JQWQsCgZuQWsPqQ2P2v2W3d1ZixQp7nH/KCODAz+CsLHa53KYvi5ORQvuXAmJQuAcVUx9f1ObmyPceJnuHxa2yxCVCNSAAmLsio6kesFRur7utA2uIqmHJuLwjxYOhaO/Qh6WxgyD0KHaZ3K4jxzX130RW1UHevVoLG/m7aBfJtAo76AAjtnaptFiGpECiBhMRRFYWHR5a+R0vxcUn6OurXFqTXqzKDhi6DpQK1TWaSgGs4Mb1cHg17H80V7hWmuy1T1z8NLIO32W3YIYU2kABIWIyYhjZPJmdjb6Hm4TSXstm2u8jJh4aNqI6ytE4xaCg17aJ3Kov17YHMOvt6djvW9tY6iqh2ubmhrKoRds7ROI0S1IAWQsBjS/HwL16/Bd4MgbifYu8GYlVD3fq1TWTyDXoe7k63WMUrq8pL658EFkHVZ2yxCVANSAAmLkH69gJ8PF6383F6anwHIToUF/SHxADh6wtjVUKeD1qmEVkLug5ptoDAX9s7ROo0QmpMCSFiEVTGJ5BaYaOjnQliQND+TkQTz+kDyEXD2hXFroGZrrVMJLel0f44C7fsGctO1zSOExqQAEmbvrys/S/MzkBYP83pD6ilwrQnj14JfM61TieqgUR/waQx5GbD/f1qnEUJTUgAJs3cw/s/m58FtrHzl5ytnYW5vuHYePIJgwq/g3UDrVKK60Ov/XBdoz2wouK5tHiE0JAWQMHs3Rn/6tayJu2M1azytSikn1JGfjAtQowGM/xU8g7VOJaqb5o+Aex3IvgzRP2idRgjNSAEkzFp6TgG/SPMzJB1Se36yLoFvM/Wyl7ssBSBuwWALnSap93d+pq4ObunyMuHcFijI1TqJqEakABJmbWX0BfIKTTT2d6VNHQ+t42gj5yr88Chcv6o2Oo/7BVx8tU4lqrPWo8HZB9Lj4chyrdNUDkWBuN3w00T4byP4biB82029TCwEUgAJM6YoCov2FTU/t7fi5ud1/4TsFPBuBI+tAicvrROJ6s7WESImqvd3fAImk7Z5KlJGkrrx66wwmNdL3feuIFvd++7SUfj6fji+WuuUohqQAkiYrai4a5y+lIWDrZ6Braz0cs8fG+BQJKCDgZ+Dg7vWiYS5CH8c7N3V2YKn1mid5t4U5qubvS4cCp80hd/fhqtnwdZZHe2asA5ePAp1ItQZcEvHwLrXrOPyn7gtG60DCFFeN5qf+1tr83NuBvw8Rb3f4VkIbKdpHGFmHNyg3ROw/SN1xKRxP3WtIHOSckJt5D60GHJS/zxeJ0ItfJoOAnuXP4+P/VktjnbNgt2fw4UD6qbAbjWrPLrQnhRAwiyl5eTzy5EkwIqbnze+pc748gyGB1/XOo0wR+2fhd2z4eJBtUm43gNaJ7q73HQ4ukItfBKj/jzu4g+tRkCr0eB9m01oDbbQ410IbA8/PQcJe+DLLvDo/2SLGCskBZAwSz8eTCS/0ESTADdaBXpoHafqxe6AA0UL2fX/DOyctc0jzJOLD7R5DPZ9BTs+rr4FkMkEcTvUouf4aigsWr9IbwMNe6nvoV43MJTyR1qT/uDbFJaOhUtH1P3yHnhNXSlbL50h1kIKIGF2SjQ/twu0vubn/BxY/YJ6v81YqHuftnmEeev4glpMn9+mXhKqHa51oj+lX4CYSLWR+Vrsn8d9GkPrMdBymFrElUeNevDEBlj7CkR/D5vfhYS98PDXMpHASkgBJMzO/thrnEnJwtHWwMDWVtj8vGU6XD2nbnPR499apxHmziNQLSRiFqq9QCMWaZunMA9OrlFHe85uAhT1uL2buohj6zFQq03F9CvZOqqTB+p0gDUvwZkN8FVXGLIAaofd+/lFtSYFkDA7kUWjPwNCa+LmYGXNz4lRsPsL9X6/T2TWl6gYnaZAzCJ1NljKCfBtUvUZkg6rRc+RpXD92p/Hg7uoRU+T/mDnVDmv3Xo0BITC0sfUXy7m9oReM6DtE+bXGG4uFAUUE+gNmkWQi53CrFzLzmeNtTY/F+bDqufVbxothkCjXlonEpbCp6FaYIC6LlBVybmq7kz/ZRf4qovai3T9GrjVgq6vwKRodWHP0GGVV/zc4N8Cntqi/j2YCmDty7DiccjLqtzXtUZJh2F+P3U2noZkBEiYlRUHL5BfaKJpgBsta1vZ6Mf2jyDlODh5Q6/3tU4jLE2XqXBitboy9AP/rLx95EwmOL9FHe058QsY89TjBjto3Fcdjan7gDYjAw7uMPR7daPYDW+qs82Sj6jHfBtXfR5Lk3UZNv0bDn4HKOoaVO2fAVsHTeJIASTMhlWv/HzpGGz/r3q/zwfgXEPbPMLy1GwN9R5U+252fgb9Pq7Y81+LVS+zxSyC9IQ/j/u1gDZj1FHN6tB8rNOpq2TXCoNl4yD1NHzzgDrbsuUQrdOZp8J82PslbPtQXYgSoNnD0P1tzYofkAJImJG9569y7nI2TnYGBrayooXLjIWwaiKYCqFRX/UbhxCVoctLagEU/QPc9w9w9bu38xVcV1dojv5enWV2g4MHtBz6Z+9NdVSnAzy9Xb0Mdn4r/PgExO9We4Ns7LVOZx4UBU7/pm7Xc/WceiwgFHq9B0Edtc2GFEDCjNxofh7Yqiau1tT8vOcLuBitblvQ9yNpyhSVJ6gT1G4HF/ap/+66v1P2cyiK+u81+gf1clpeetEndOpig61Hq6tOa/ibf6m5+MCYlbDlPdj2gbpcwMWD6iwxzyCt01VvKSfgt2lwbrP6sbMvdHsTWo2qNmstSQEkzMLV7Hx+PZIMwMh2VvSNJ/UMbJ6u3u/5H3AL0DaPsGw6nToKFDkM9s+Fzi+Co2fpnpudCoeXqoVPyrE/j3vUUVdnbjVCvW9u9AZ48DV1q5kfn1SLu6+6qusFNeypdbrqJ+eq+j3rwFxQjGpvV4fn1H9XDm5apyuhzAVQcHAwEyZMYNy4cdSpY4b/mIVZWhF1gXyjiea13GhhLc3PJpO64GFhrtoU2nq01omENWjYE3ybqUXMvm/hvldu/1hjYdEls+/h1K/q7CkAGwdoMkD9Nxvcpdr8xn9PGnRXL4ktG6suR7FoqPpD/f5/ln4FaktmLFCLns3TITdNPda4n7pWmVddTaPdTpn/Vb700kusWrWKunXr0r17dxYvXkxeXl5lZBMCUJufI4tXfrai0Z8D/4P4XeqO1v0/lUtfomrodOqMMIC9cyA/++bHXDkLG9+Gmc1h0RB19pipQG2k7vsRvHQKHvlGXaXcEoqfGzwCYfyv0O4p9ePtH8H3gyArRdNYmjuzEeZ0gl//Ty1+fJvBY6th+MJqW/wA6BRFUcrzxEOHDjF37lwiIyMpLCxk5MiRTJgwgTZt2lR0xiqXkZGBu7s76enpuLlVryE7a7T77BVGfLMHZzsDe197CBd7K/htKy0eZkdAfhb0/gDaP611ImFNjIXweZg6c6vXe9DhWbUQOr5KvcQVt/PPxzp6QehwtbfDv7lmkavckeWwehIUZKsbsQ6ZVy0ae6tU6hm1wfmPderHTjXUPdXajNVsVKwsP7/LXQDdUFBQwOzZs/nHP/5BQUEBzZs3Z/LkyYwfP95spylLAVS9vBAZzc+HLjKyfR2mD26hdZzKpyjwwyNw9ncI7KD+xmlJv0UL83BgLvzyorrlSoOH4OiPakEOoNND/YfUS1wNe4ONnbZZtXL5lLp69OWToDPAQ2+pe6uZ6c++UrueBls/UBeuNBWqm9K2exru+z9w9NA0Wll+fpf7u2pBQQFLly5lwIABvPTSS4SHh/Ptt98ydOhQXnvtNUaNGlWq88yePZuQkBAcHBwICwtj+/btt33sjz/+SPfu3fHx8cHNzY2IiAjWrVt30+NWrFhB06ZNsbe3p2nTpqxcubK8b1No7EpWHr8dLVr5uZ2V9JwdilSLH4O9uk+RFD9CC6Ej1ZGNzIvqwnX5WerljG5vwovHYNQyaDrQeosfAJ9G8OQmaDFUbfjd8AYsGa0WCJbIZFQL41lt1FmCpkJo0AOe2wO9pmte/JRVmceoDh48yLx584iMjMRgMDBmzBg++eQTGjf+c5XMHj160LVr17uea8mSJUyZMoXZs2fTqVMnvvrqK3r37s3x48dv2WC9bds2unfvzvTp0/Hw8GDevHn079+fvXv30rp1awB2797NsGHD+Pe//83gwYNZuXIlQ4cOZceOHbRv376sb1dobHnUBQqMCi1ru9O8lhU0P2deUqeOAtz/Kng30DaPsF62DurMww1vQsh96mhPUEfLH90oKztndUZYnQ7w26tw8hd14dKh30FAS63TVZzz29TvTZeOqh97N4SeM9TRQTNV5ktgBoOB7t278/jjjzNo0CBsbW9ejyU7O5vnn3+eefPm3fFc7du3p02bNsyZM6f4WJMmTRg0aBAzZswoVZ5mzZoxbNgw3nzzTQCGDRtGRkYGv/76a/FjevXqhaenJ5GRkaU6p1wCqx4UReGB/24h9koO7z3cguHWMAK0ZLS6cFxAKDyxSWaXCGFOEg/C0rGQHq+O4Pb9L7R5TOtU9+bqeVj/ulrYgbpdyP3/hLaPg6H6rcdWqZfAzp07x2+//caQIUNuWfwAODs737X4yc/PJyoqih49epQ43qNHD3bt2lWqLCaTiczMTLy8/lw+fffu3Teds2fPnqU+p6g+dp+9QuyVHFzsbegfagUrPx/7SS1+9DYw8AspfoQwN7XawNNboUFPdY+z1S/ATxMhP0frZGWXlwkb34Iv2qnFj84AbZ+ESTHQ4ZlqWfyUVZm/w6akpJCcnHzT5aS9e/diMBgIDw8v1XlSU1MxGo34+ZVcat3Pz4/k5ORSneOjjz4iOzuboUOHFh9LTk4u8znz8vJKTOXPyMgo1euLyrXwLys/O1v6zK+cq+ru06AuPudvBc3eQlgiJy8YsRh2fgKb3oWYHyApRr0kVqOe1unuzmSCQ4vg93cg65J6rO4D6hYgvk20zVbByjwCNHHiRBISEm46npiYyMSJE8sc4O8zxRRFKdXsscjISN566y2WLFmCr6/vPZ1zxowZuLu7F98CAwPL8A5EZUjNymP9saKVn9tbwaWv36ZB9mXwaQxd77DwnBCi+tPr1UUSH1sFzj5q38xX96nLCFRn8XvUjV9XTVSLH6+6MDxS3Q7EwoofKEcBdPz48Vuu9dO6dWuOHz9e6vN4e3tjMBhuGplJSUm5aQTn75YsWcLjjz/O0qVLeeihkg1Y/v7+ZT7ntGnTSE9PL77dqsATVWvZAbX5OTTQg2Y1Lbz5+fR6OLwY0MGAz2WjRSEsRUhXdfXoOh0hP1OdMv/bP9VVk6uTtARYPgHm9lRHq+zdoPu/1dldjftYbON7mQsge3t7Ll26dNPxpKQkbGxKf5nCzs6OsLAwNmzYUOL4hg0b6Njx9otJRUZGMm7cOBYtWkTfvn1v+nxERMRN51y/fv0dz2lvb4+bm1uJm9COyaSweL96+WuUpTc+52bAL1PU+x2eg8C2msYRQlQwtwAY+zN0nKR+vOcLmN8XMi5qmwvUxS03T4fPw+HoCkCnLmL4wkHoNMnifxkrc2NF9+7dmTZtGqtWrcLdXf3NPC0tjX/+85907969TOeaOnUqY8aMITw8nIiICL7++mvi4+N55plnAHVkJjExke+++w5Qi5/HHnuMTz/9lA4dOhSP9Dg6OhZnmTx5Ml27duX9999n4MCBrFq1io0bN7Jjx46yvlWhkV1nrxB3JQdXexv6hVr45p8b/wUZieAZDA++rnUaIURlMNioe2IFtoefnoOEvfBlF3jkW6j3QNXnURQ4sgw2/Etd5wkgqJPa5xMQWvV5NFLmAuijjz6ia9euBAUFFa+9ExMTg5+fH99//32ZzjVs2DCuXLnCO++8Q1JSEs2bN2ft2rUEBan7PSUlJREfH1/8+K+++orCwkImTpxYot9o7NixzJ8/H4COHTuyePFiXn/9dd544w3q1avHkiVLZA0gM7JoXxwAg1rXwsnOgpufz29XFxUDGDAL7Jy0zSOEqFxN+oFfU/VSWPIR+H4wPPBP6PJy1S14eiEKfvsHXNivfuxRR73c1XSgxV7qup1ybYWRnZ3NwoULOXToEI6OjrRs2ZIRI0bcdlq8uZF1gLSTkplLxxmbKDQp/Dq5C00CLPTvPz8H5nSEa+chbJy62akQwjoUXIdf/wEHF6gf138IHv5GnUFWWTKS4Pe31ZXmQd1kuctUiHheXfTSQpTl53e5fr12dnbmqaeeKlc4Ie5kedQFCk0Kret4WG7xA7D5P2rx41oTur+jdRohRFWydYQBn6mrR/8yVd1N/csuMHQB1C7dUjKlVnAddn8O2z9RN24FdZuTbm+q/UlWrNzXF44fP058fDz5+fkljg8YMOCeQwnrZDIpLN6nzsCz6H2/LkTBntnq/f4z1ZVVhRDWp9VI8G+pXhK7ehbm9oKe06Hdk/d+OUpR1Gn3699QV6YGqN0Oer8HtcLuPbsFKHMBdO7cOQYPHsyRI0fQ6XTcuIJ2Y50do9FYsQmF1dhxJpX4qzm4OtjQr6WFrvxcmKeusaGY1A0UG/bUOpEQQkv+zeGpLbD6ebVg+fUVSNijXha3dy3fOZMOqWuLxe1UP3arBQ+9DS0etbo+nzspc9fV5MmTCQkJ4dKlSzg5OXHs2DG2bdtGeHg4W7ZsqYSIwlos2qv+lvJw61o42hk0TlNJtn8El0+Akzf0ek/rNEKI6sDBDYYsUDcX1duoU9K/eRBSTpTtPFkp6vYbX92nFj82jnDfq/D8fmg5RIqfvynzCNDu3bvZtGkTPj4+6PV69Ho9nTt3ZsaMGUyaNIno6OjKyCksXEpGLhtPqOtLjWwfpHGaSpJ8VC2AAPp8CM41tM0jhKg+dDqIeE69PLVsHKSeVoug/p9Cy6F3fm5hPuz9ErZ+oC64CND8Uej+NrjXrvTo5qrMI0BGoxEXFxdAXc354kV1DYGgoCBOnTpVsemE1VhW1PwcFuRJI/9yDvtWZ8ZC9dKXqRAa94Nmg7VOJISojuq0h2e2q/tvFeTAj0/CLy9CQe7Nj1UUOLkWZreHDW+oxU/N1jBhPTz6Pyl+7qLMI0DNmzfn8OHD1K1bl/bt2/PBBx9gZ2fH119/Td26dSsjo7BwJpNCZNHGpyMstfl59+fqEvMO7tD3IxmKFkLcnrM3jF6hjuhsfV9dLyzxoLqhqmfRCPml47BuGpzbon7s4gcPvQUth1fdmkJmrswF0Ouvv052tjqV7t1336Vfv3506dKFGjVqsGTJkgoPKCzftj8uc+HaddwcbOjX0gKnZaaegS0z1Ps9p4Orv7Z5hBDVn94AD0yD2m3hxyfUX6C+6qr+AhW/Wy2KFBMY7CFiorqmT3mbpq1UmQugnj3/nLVSt25djh8/ztWrV/H09CzVLu5C/F1x83Ob2jjYWljzs8mkzu4ozIV6D0KrUVonEkKYkwYPqRuqLhsLiVGw4vE/P9dkgLrFhmewZvHMWZnGyQoLC7GxseHo0aMljnt5eUnxI8rlUkYuv59MAWBUewu8/HXgf+pva7bO0G+mXPoSQpSdRyCM/w3aPa1+7NcCxv4Cw76X4ucelGkEyMbGhqCgIFnrR1SYpfsTMJoU2gZ70sDPwoZv0+Jh41vq/Yfe+vPavRBClJWNHfT5ALq+Ak41pM+nApT5b/D1119n2rRpXL16tTLyCCtiNCks3q+u/Gxxzc+KAj9PhvwsqBMBbZ/QOpEQwhK4+EjxU0HK3AP02WefcebMGWrWrElQUBDOzs4lPn/w4MEKCycs27bTl0lMu467oy19WlhY83PMIji7SW1QHDBLvmEJIUQ1U+YCaNCgQZUQQ1ijRUVT3x+xtObnzGR1eiqoszi8G2ibRwghxE3KXAD961//qowcwsokp+eyqaj5eWT7QI3TVCBFgTUvQW46BLSCiBe0TiSEEOIWZFxeaGJJUfNzu2Av6vtaUPPz8Z/g5C/qfj4DPwdDmX/HEEIIUQXK/N1Zr9ffccq7zBATd2M0KSzZr17+GmlJU99zrsLaV9T7naeCfwtt8wghhLitMhdAK1euLPFxQUEB0dHRLFiwgLfffrvCggnLtfV0ChfTc/F0sqVXcwtaFfm3VyH7Mvg0hq4va51GCCHEHZS5ABo4cOBNxx599FGaNWvGkiVLePzxx2/xLCH+dGPlZ4tqfj69Hg4vAZ0eBn4BNvZaJxJCCHEHFdYD1L59ezZu3FhRpxMW6mLa9eLm5+GWsvZPbgb8MkW93+E5qB2uaRwhhBB3VyEF0PXr15k1axa1a9euiNMJC7ZkfwImBdqHeFHf10XrOBVjw5uQkQieIfDAa1qnEUIIUQplvgT2901PFUUhMzMTJycnfvjhhwoNJyxLodHEkqKVny2m+fn8doiap94fMAvsnLTNI4QQolTKXAB98sknJQogvV6Pj48P7du3x9PTs0LDCcuy5dRlkjNy8XK2s4zm5/wcWF20zk/YeAjpom0eIYQQpVbmAmjcuHGVEENYgxsrPz8aVht7Gwtoft78H7h2HtxqQfd3tE4jhBCiDMrcAzRv3jyWLVt20/Fly5axYMGCCgklLE9i2nW2nCpqfm5rASs/XzgAe2ar9/vNBAc3TeMIIYQomzIXQO+99x7e3t43Hff19WX69OkVEkpYniX74jEpEFG3BnV9zLz5uTAPVk0ExQQth0HDHlonEkIIUUZlLoDi4uIICQm56XhQUBDx8fEVEkpYlkKjiSUHLKj5edt/4fJJcPaBXu9pnUYIIUQ5lLkA8vX15fDhwzcdP3ToEDVq1KiQUMKybDqZwqWMPGo429GzmZk3PycfgR0fq/f7fAhOXtrmEUIIUS5lLoCGDx/OpEmT2Lx5M0ajEaPRyKZNm5g8eTLDhw+vjIzCzP21+dnOxoz33zUWwqrnwVQIjftB00FaJxJCCFFOZZ4F9u677xIXF0e3bt2wsVGfbjKZeOyxx6QHSNwk4WoOW09fBmCEua/8vHsWJMWAgzv0/QjusCmwEEKI6q3MBZCdnR1Llizh3XffJSYmBkdHR1q0aEFQUFBl5BNmbumBBBQFOtWvQbC3s9Zxyi/1D9g8Q73fcwa4mvmlPCGEsHJlLoBuaNCgAQ0aNKjILMLCFPx15ed2Zlwgm0zqgofGPKjXDVqN1DqREEKIe1TmhoxHH32U9967eebLhx9+yJAhQyoklLAMv59IISUzD28XO7o39dM6Tvnt/xbid4OdC/SfKZe+hBDCApS5ANq6dSt9+/a96XivXr3Ytm1bhYQSluHP5udA821+vhYHG99S7z/0FniYeR+TEEIIoBwFUFZWFnZ2djcdt7W1JSMjo0JCCfOXcDWH7X/caH4205WfFQV+ngwF2VCnI4Q/rnUiIYQQFaTMBVDz5s1ZsmTJTccXL15M06ZNKySUMH+L98ejKNClgTdBNcy0+TlmIZzbDDYO6k7vejMdxRJCCHGTMjdBv/HGGzzyyCOcPXuWBx98EIDff/+dRYsWsXz58goPKMxPgdHE0gMXABhprlPfM5Nh3T/V+/dPA+/62uYRQghRocpcAA0YMICffvqJ6dOns3z5chwdHQkNDWXTpk24ucmGkAI2Hr/E5cw8vF3secgcm58VBda8BLnpENAKIp7XOpEQQogKVq5p8H379i1uhE5LS2PhwoVMmTKFQ4cOYTQaKzSgMD83mp+HhtfG1mCGl42OrYSTv4DeBgZ+AYZyrxYhhBCimir3T6dNmzYxevRoatasyeeff06fPn04cOBARWYTZij+Sg7b/0hFpzPTlZ+zr8DaV9T7XV4C/+ba5hFCCFEpyvSr7YULF5g/fz5z584lOzuboUOHUlBQwIoVK6QBWgAQuV8d/enSwIdALyeN05TDb69CTir4NIEuL2udRgghRCUp9QhQnz59aNq0KcePH2fWrFlcvHiRWbNmVWY2YWbyC00sO3Bj5WcznPp+eh0cWQo6vXrpy+bm5R6EEEJYhlKPAK1fv55Jkybx7LPPyhYY4pY2HL9EalY+Pq72dGtiZs3Puenw8xT1fofnoHaYpnGEEEJUrlKPAG3fvp3MzEzCw8Np3749n3/+OZcvX67MbMLMRBY1Pw8LDzS/5ucNb0LmRfCqCw+8pnUaIYQQlazUP6UiIiL45ptvSEpK4umnn2bx4sXUqlULk8nEhg0byMzMrMycopqLTc1mxxm1+XlYWzO7/HV+G0TNV+8PmAV2Zti7JIQQokzK/Gu6k5MTEyZMYMeOHRw5coSXXnqJ9957D19fXwYMGFAZGYUZuNH83NXcmp/zs9Wd3gHCJ0BwZ23zCCGEqBL3dJ2iUaNGfPDBB1y4cIHIyMiKyiTMTH6hieU3Vn5ubyZT36+nwfntsHoSXIsFt9rw0NtapxJCCFFFKmSFN4PBwKBBgxg0aFBFnE6YmfXHk7mSnY+fmz3dGvtqHackRVG3tUg+DEmHIfmQ+mdaXMnH9Z8JDrKSuRBCWAtZ4lbcs0V7/2x+ttGy+dlkgmvnIenQXwqew5B9m2Z99zoQ0BKaDYYG3as2qxBCCE1JASTuyfnUbHadvaI2P1flys+F+XD5ZMlCJ/ko5N+iGV+nhxoN1GLHvyUEhIJ/C3Dyqrq8QgghqhUpgMQ9uTH1/f6GPtTycKycF8nLgkvHioqdotGdlBNgzL/5sQZ78GtaVOi0BP9Q8GsmM7uEEEKUIAWQKLe8QiPLo240PwdVzEmzr/zZp3NjdOfKGUC5+bH27upITvHITkvwbggG24rJIoQQwmJJASTKbd2xS1zNzsffzYEHGvmU7cmKAukJJQud5MOQkXjrx7v4lyx0/FuCZzDodPf8PoQQQlgfKYBEuS3aq86kGtb2Ls3PJiOk/lHyElbyEbh+7daP9wxRi5yAUPUSVkBLcKlms8uEEEKYNSmARLmcvZzFnnNX0f995eeCXEg5fnNzcuH1m0+itwGfxiVHdfybg4N71b0RIYQQVkkKIFEukXvjcSWH0UHp1Dwx78+C5/JJUIw3P8HWCfyal7yM5dMEbB2qPrwQQgirJwWQKLPc6zn0OTCB1x1OQDLq7a8cvf425bwl1KgHeoMWcYUQQoibSAEkyixm3QI6cAIAxa02ur+O6gSEglstaU4WQghRrUkBJMpEURScj/4AwIHgpwgf96HGiYQQQoiy03DfAmGODkXvo0XhUYyKjga9J2odRwghhCgXKYBEmaRu+QqA0+6dcPcL1jaMEEIIUU5SAIlSu5ByhfD03wBw7/KkxmmEEEKI8pMCSJTagV8X4KHLJtXgS82w/lrHEUIIIcpNCiBRKll5hQSeWwJARtMRMqVdCCGEWZMCSJTK71u3EKY7iRE9wQ89o3UcIYQQ4p5IASTuymRSKNw/D4BE3/vQu9fUOJEQQghxb6QAEne17Xg8D+VvAsD3/mc1TiOEEELcOymAxF2d3PQ97roc0uz8cWj8kNZxhBBCiHsmBZC4o9OXMglPXaV+EDZWmp+FEEJYBCmAxB2t/X0T4frTGDHg0XGC1nGEEEKICqF5ATR79mxCQkJwcHAgLCyM7du33/axSUlJjBw5kkaNGqHX65kyZcpNj5k/fz46ne6mW25ubiW+C8t0LTsf75MLAUiv8xC4+mucSAghhKgYmhZAS5YsYcqUKbz22mtER0fTpUsXevfuTXx8/C0fn5eXh4+PD6+99hqhoaG3Pa+bmxtJSUklbg4ODpX1NizWsj2nGKBTC1LPrk9rnEYIIYSoOJoWQB9//DGPP/44TzzxBE2aNGHmzJkEBgYyZ86cWz4+ODiYTz/9lMceewx3d/fbnlen0+Hv71/iJsqmwGji0u7FuOlyyHKqja7uA1pHEkIIISqMZgVQfn4+UVFR9OjRo8TxHj16sGvXrns6d1ZWFkFBQdSuXZt+/foRHR19x8fn5eWRkZFR4mbt1h1Lpm++uu+XQ/vxoNf8aqkQQghRYTT7qZaamorRaMTPz6/EcT8/P5KTk8t93saNGzN//nxWr15NZGQkDg4OdOrUiT/++OO2z5kxYwbu7u7Ft8DAwHK/vqX4fcsm2ujPYNQZsGkzRus4QgghRIXS/Nd6nU5X4mNFUW46VhYdOnRg9OjRhIaG0qVLF5YuXUrDhg2ZNWvWbZ8zbdo00tPTi28JCQnlfn1LcCghjVYpPwFQ0KAPuPrd+QlCCCGEmbHR6oW9vb0xGAw3jfakpKTcNCp0L/R6PW3btr3jCJC9vT329vYV9prmbuH247xu2AGAQ3uZ+i6EEMLyaDYCZGdnR1hYGBs2bChxfMOGDXTs2LHCXkdRFGJiYggICKiwc1qySxm56I+vxE13nTzXOhByv9aRhBBCiAqn2QgQwNSpUxkzZgzh4eFERETw9ddfEx8fzzPPqLuNT5s2jcTERL777rvi58TExABqo/Ply5eJiYnBzs6Opk2bAvD222/ToUMHGjRoQEZGBp999hkxMTF88cUXVf7+zNEPe+IYrv8dAPv2E6T5WQghhEXStAAaNmwYV65c4Z133iEpKYnmzZuzdu1agoKCAHXhw7+vCdS6devi+1FRUSxatIigoCBiY2MBSEtL46mnniI5ORl3d3dat27Ntm3baNeuXZW9L3OVW2Bk/56tvKQ/i0lni77VaK0jCSGEEJVCpyiKonWI6iYjIwN3d3fS09Nxc3PTOk6VWbo/gbxVUxhjsxFTk0Hohy3QOpIQQghRamX5+S3XNwSg9kot2nGcQYadAOjbjtc4kRBCCFF5pAASAOw5d5VGqRtw1V3H6BECwV21jiSEEEJUGimABABzd55npEFtfja0lZWfhRBCWDb5KSeIv5JD0sk9hOrPoehtodUorSMJIYQQlUoKIMGC3bGM0G8CQNekPzh7a5xICCGEqFyaToMX2svKK+SX/af5vaj5mXBpfhZCCGH5ZATIyi0/kMCDhdtw0eWi1KgPwV20jiSEEEJUOimArJjJpDB/VywjDEWXv8LGwT1sRCuEEEKYCymArNjmUym4XD1KS/15FIMdhI7UOpIQQghRJaQAsmLzdsYWT33XNRkAzjU0TiSEEEJUDSmArNTpS5nEnIlnoGGXekCan4UQQlgRKYCs1Lyd5xlo2IWzLg+8G0JQJ60jCSGEEFVGpsFboWvZ+fx48AI/Fl3+QpqfhRBCWBkZAbJCkfvjaWQ8QzN9HIrBHkJHaB1JCCGEqFJSAFmZAqOJ73bF/dn83HQgOHlpnEoIIYSoWlIAWZnfjiaTnXGVgTa71QPS/CyEEMIKSQFkZdTm5504kgfejaBOhNaRhBBCiConBZAViUlI42D8NUbZqCs/S/OzEEIIayUFkBWZt/M8rXRnaaKLA4M9hA7XOpIQQgihCSmArMSljFzWHE4qbn6m2WBpfhZCCGG1pACyEt/vjsPRlM1Amz3qAWl+FkIIYcWkALICuQVGFu2LZ5BhB/bkgU9jCGyvdSwhhBBCM1IAWYHVMRe5mp3HWLvN6oGw8dL8LIQQwqpJAWThFEVh7s7ztNH9QX0lDmwcIHSY1rGEEEIITUkBZOF2n7vCyeRMxtgWjf40exgcPbUNJYQQQmhMCiALN29nLG5k0c8gKz8LIYQQN0gBZMHirmSz8cQlBht2Yqvkg29TqN1W61hCCCGE5qQAsmALdsWhKApPOm1VD0jzsxBCCAFIAWSxMnMLWHoggTDdaWoXxIKNI7QcqnUsIYQQolqQAshCLY+6QFZeIU87b1MPNH8EHD00zSSEEEJUF1IAWSCTSWHBLrX5+UHTLvVg2DhNMwkhhBDViRRAFmjzqRRir+QwymEXNqY88GsOtcO1jiWEEEJUG1IAWaC5O88DCo87bFEPhI2T5mchhBDiL6QAsjCnkjPZeeYK7fSn8M6NBVsnaX4WQggh/kYKIAszf9d5AF6qUdT70/xhcHDXMJEQQghR/UgBZEGuZufz48FEPMikbXbR7K+wCdqGEkIIIaohKYAsSOS+ePIKTUz02o/elA/+LaBWG61jCSGEENWOFEAWosBo4vvdcYDCcN3v6kFZ+VkIIYS4JSmALMSvR5NJzsilu/NZXLPPg60ztBiidSwhhBCiWpICyELM21nU/Oy1Uz3Q4hFwcNMwkRBCCFF9SQFkAaLjrxEdn4afIYtGVzerB8PGaxtKCCGEqMakALIA83bGAvBa7Rh0xnwICJXmZyGEEOIOpAAyc8npuaw9kgQo9Mz9TT0ooz9CCCHEHUkBZOZ+2BNHoUlhXMAF7NPPgZ0LtHhU61hCCCFEtSYFkBnLLTCycG8cAE+7bFUPtngU7F01TCWEEEJUf1IAmbFVMYlcyymgmXs+/okb1INy+UsIIYS4KymAzJSiKMXNz2/UjkZnKoCaraFmK01zCSGEEOZACiAztfvsFU4mZ+Jkp6ftldXqQRn9EUIIIUpFCiAzNbdo9OflBikY0s6DnSs0f0TbUEIIIYSZkALIDMVdyeb3k5cAGKor6v1pOQTsXTRMJYQQQpgPKYDM0PxdsSgK9K9vg8s5WftHCCGEKCspgMxMZm4Byw5cAOBF7wNgKoBaYRDQUuNkQgghhPmQAsjMLI+6QFZeIfW9HQmJW6YeDBunaSYhhBDC3EgBZEaMJoX5u2IBeLXxZXTXzoO9mzQ/CyGEEGUkBZAZ2XwyhbgrObg52HB/1i/qwZZDwc5Z22BCCCGEmZECyIzM23UegMdbO2Nzeq16UC5/CSGEEGUmBZCZOJmcwc4zVzDodYxz2gWmQqgVDv4ttI4mhBBCmB0pgMzE/KKFD3s19cH9+EL1YLhMfRdCCCHKQwogM3A1O5+V0YkATApJhGuxYO8OzR7WNpgQQghhpqQAMgOR++LJKzTRopY7DS8sVw+GDgM7J22DCSGEEGZKCqBqrsBo4vvdcQA8G+aM7tSv6iek+VkIIYQoNymAqrlfjyaTnJGLt4s9PfI3qs3PtduBXzOtowkhhBBmSwqgam7uDnXq+5j2gdjEfKcelOZnIYQQ4p5IAVSNRcdfIyYhDTuDnrF+5yAtHhzcodlgraMJIYQQZk0KoGpsXtHU9/6hNfE4/oN6MHQE2DpqF0oIIYSwAFIAVVPJ6bmsPZIEwJOtHECan4UQQogKIwVQNfX9nlgKTQrtQrxonLQaFCMEdgDfJlpHE0IIIcyeFEDVUG6BkUV74wGY0DEQDi5QPyHNz0IIIUSFkAKoGvopOpFrOQXU9nSku91xSE8ABw9oOlDraEIIIYRFkAKomlEUpbj5eWxEMIaD89VPtBopzc9CCCFEBdG8AJo9ezYhISE4ODgQFhbG9u3bb/vYpKQkRo4cSaNGjdDr9UyZMuWWj1uxYgVNmzbF3t6epk2bsnLlykpKX/F2n73CqUuZONkZGNbYBk7/pn5Cmp+FEEKICqNpAbRkyRKmTJnCa6+9RnR0NF26dKF3797Ex8ff8vF5eXn4+Pjw2muvERoaesvH7N69m2HDhjFmzBgOHTrEmDFjGDp0KHv37q3Mt1Jh5u5UFz58NKw2bscj1ebnOh3Bp5HGyYQQQgjLoVMURdHqxdu3b0+bNm2YM2dO8bEmTZowaNAgZsyYccfn3n///bRq1YqZM2eWOD5s2DAyMjL49ddfi4/16tULT09PIiMjS5UrIyMDd3d30tPTcXNzK/0bukdxV7K5/79bUBT4/cXO1FvYETIuwMPfQMuhVZZDCCGEMEdl+fmt2QhQfn4+UVFR9OjRo8TxHj16sGvXrnKfd/fu3Teds2fPnnc8Z15eHhkZGSVuWpi/KxZFgfsb+VAvfY9a/Dh6QpMBmuQRQgghLJVmBVBqaipGoxE/P78Sx/38/EhOTi73eZOTk8t8zhkzZuDu7l58CwwMLPfrl1dmbgHLDlwAYEKnEDgwT/1E6EiwdajyPEIIIYQl07wJWqfTlfhYUZSbjlX2OadNm0Z6enrxLSEh4Z5evzyWHbhAVl4h9X1d6OKbC3+sUz8hzc9CCCFEhbPR6oW9vb0xGAw3jcykpKTcNIJTFv7+/mU+p729Pfb29uV+zXtlNCks2B0LwLiOweiifwDFBEGdwaehZrmEEEIIS6XZCJCdnR1hYWFs2LChxPENGzbQsWPHcp83IiLipnOuX7/+ns5Z2TadTCHuSg7ujrY83MoPor9XPyErPwshhBCVQrMRIICpU6cyZswYwsPDiYiI4OuvvyY+Pp5nnnkGUC9NJSYm8t133xU/JyYmBoCsrCwuX75MTEwMdnZ2NG3aFIDJkyfTtWtX3n//fQYOHMiqVavYuHEjO3bsqPL3V1rziqa+D28XiFPcZshIBEcvaNJf42RCCCGEZdK0ABo2bBhXrlzhnXfeISkpiebNm7N27VqCgoIAdeHDv68J1Lp16+L7UVFRLFq0iKCgIGJjYwHo2LEjixcv5vXXX+eNN96gXr16LFmyhPbt21fZ+yqLk8kZ7Dp7BYNex2MRwbDmX+onWo0EG+0uywkhhBCWTNN1gKqrqlwH6B/LD7PkQAJ9WwTwRV8f+LSl2v/zfBR416/U1xZCCCEsiVmsAyTganY+P8UkAjC+U7Da+6OYILiLFD9CCCFEJZICSEOR++LJKzTRopY7YYGucLCo10man4UQQohKJQWQRgqMJr4rmvo+oXMwuj/WQ2YSONWAxv20DSeEEEJYOCmANLL2SBKXMvLwcbWnb4uaf6783GqUND8LIYQQlUwKII3M2xkLwOj2QdhlJsCZjeonZOVnIYQQotJJAaSBg/HXiElIw86gZ1SHOkULHyoQch/UqKd1PCGEEMLiSQGkgRujPwNa1cTbUQ8Hi1Z+ltEfIYQQokpIAVTFktKv8+uRJKBo6vvp3yArGZx9pPlZCCGEqCJSAFWx73fHUWhSaB/iRbOa7n9rfrbTNpwQQghhJaQAqkK5BUYi96lbe4zvFALXYuHsJvWTYWO1CyaEEEJYGSmAqtCaw0lcyymgtqcj3Zv6FS18qEDdB8CrrtbxhBBCCKuh6Wao1mZQ61o429tgUhQMSiFE/6B+QpqfhRBCiColBVAVMuh19Grur35wfBVkXQJnX2jcV9tgQgghhJWRS2BaiZqv/tl6NBhsNY0ihBBCWBspgLRw9bw0PwshhBAakgJICwcXqH/WexA8gzWNIoQQQlgjKYCqWmH+X5qfx2ubRQghhLBSUgBVtVNrIPsyuPhBo95apxFCCCGskhRAVa24+XmMND8LIYQQGpECqCpdOQvntgA6aPOY1mmEEEIIqyXrAFWla7Hg4g/+zcEzSOs0QgghhNWSAqgq1e8GLx6FnKtaJxFCCCGsmlwCq2oGW3D10zqFEEIIYdWkABJCCCGE1ZECSAghhBBWRwogIYQQQlgdKYCEEEIIYXWkABJCCCGE1ZECSAghhBBWRwogIYQQQlgdKYCEEEIIYXWkABJCCCGE1ZECSAghhBBWRwogIYQQQlgdKYCEEEIIYXWkABJCCCGE1bHROkB1pCgKABkZGRonEUIIIURp3fi5fePn+J1IAXQLmZmZAAQGBmqcRAghhBBllZmZibu7+x0fo1NKUyZZGZPJxMWLF3F1dUWn01XouTMyMggMDCQhIQE3N7cKPbcoO/l6VC/y9ahe5OtR/cjX5M4URSEzM5OaNWui19+5y0dGgG5Br9dTu3btSn0NNzc3+cdbjcjXo3qRr0f1Il+P6ke+Jrd3t5GfG6QJWgghhBBWRwogIYQQQlgdKYCqmL29Pf/617+wt7fXOopAvh7VjXw9qhf5elQ/8jWpONIELYQQQgirIyNAQgghhLA6UgAJIYQQwupIASSEEEIIqyMFUBWaPXs2ISEhODg4EBYWxvbt27WOZLVmzJhB27ZtcXV1xdfXl0GDBnHq1CmtYwnUr41Op2PKlClaR7FqiYmJjB49mho1auDk5ESrVq2IiorSOpZVKiws5PXXXyckJARHR0fq1q3LO++8g8lk0jqaWZMCqIosWbKEKVOm8NprrxEdHU2XLl3o3bs38fHxWkezSlu3bmXixIns2bOHDRs2UFhYSI8ePcjOztY6mlXbv38/X3/9NS1bttQ6ilW7du0anTp1wtbWll9//ZXjx4/z0Ucf4eHhoXU0q/T+++/z5Zdf8vnnn3PixAk++OADPvzwQ2bNmqV1NLMms8CqSPv27WnTpg1z5swpPtakSRMGDRrEjBkzNEwmAC5fvoyvry9bt26la9euWsexSllZWbRp04bZs2fz7rvv0qpVK2bOnKl1LKv06quvsnPnThmlrib69euHn58f//vf/4qPPfLIIzg5OfH9999rmMy8yQhQFcjPzycqKooePXqUON6jRw927dqlUSrxV+np6QB4eXlpnMR6TZw4kb59+/LQQw9pHcXqrV69mvDwcIYMGYKvry+tW7fmm2++0TqW1ercuTO///47p0+fBuDQoUPs2LGDPn36aJzMvMleYFUgNTUVo9GIn59fieN+fn4kJydrlErcoCgKU6dOpXPnzjRv3lzrOFZp8eLFHDx4kP3792sdRQDnzp1jzpw5TJ06lX/+85/s27ePSZMmYW9vz2OPPaZ1PKvzj3/8g/T0dBo3bozBYMBoNPKf//yHESNGaB3NrEkBVIX+vrO8oigVvtu8KLvnn3+ew4cPs2PHDq2jWKWEhAQmT57M+vXrcXBw0DqOAEwmE+Hh4UyfPh2A1q1bc+zYMebMmSMFkAaWLFnCDz/8wKJFi2jWrBkxMTFMmTKFmjVrMnbsWK3jmS0pgKqAt7c3BoPhptGelJSUm0aFRNV64YUXWL16Ndu2baN27dpax7FKUVFRpKSkEBYWVnzMaDSybds2Pv/8c/Ly8jAYDBomtD4BAQE0bdq0xLEmTZqwYsUKjRJZt1deeYVXX32V4cOHA9CiRQvi4uKYMWOGFED3QHqAqoCdnR1hYWFs2LChxPENGzbQsWNHjVJZN0VReP755/nxxx/ZtGkTISEhWkeyWt26dePIkSPExMQU38LDwxk1ahQxMTFS/GigU6dONy0Lcfr0aYKCgjRKZN1ycnLQ60v+uDYYDDIN/h7JCFAVmTp1KmPGjCE8PJyIiAi+/vpr4uPjeeaZZ7SOZpUmTpzIokWLWLVqFa6ursWjc+7u7jg6Omqczrq4urre1Hvl7OxMjRo1pCdLIy+++CIdO3Zk+vTpDB06lH379vH111/z9ddfax3NKvXv35///Oc/1KlTh2bNmhEdHc3HH3/MhAkTtI5m1mQafBWaPXs2H3zwAUlJSTRv3pxPPvlEplxr5Ha9V/PmzWPcuHFVG0bc5P7775dp8Br75ZdfmDZtGn/88QchISFMnTqVJ598UutYVikzM5M33niDlStXkpKSQs2aNRkxYgRvvvkmdnZ2WsczW1IACSGEEMLqSA+QEEIIIayOFEBCCCGEsDpSAAkhhBDC6kgBJIQQQgirIwWQEEIIIayOFEBCCCGEsDpSAAkhhBDC6kgBJIQQQgirIwWQEEKUgk6n46efftI6hhCigkgBJISo9saNG4dOp7vp1qtXL62jCSHMlGyGKoQwC7169WLevHkljtnb22uURghh7mQESAhhFuzt7fH39y9x8/T0BNTLU3PmzKF37944OjoSEhLCsmXLSjz/yJEjPPjggzg6OlKjRg2eeuopsrKySjxm7ty5NGvWDHt7ewICAnj++edLfD41NZXBgwfj5OREgwYNWL16deW+aSFEpZECSAhhEd544w0eeeQRDh06xOjRoxkxYgQnTpwAICcnh169euHp6cn+/ftZtmwZGzduLFHgzJkzh4kTJ/LUU09x5MgRVq9eTf369Uu8xttvv83QoUM5fPgwffr0YdSoUVy9erVK36cQooIoQghRzY0dO1YxGAyKs7Nzids777yjKIqiAMozzzxT4jnt27dXnn32WUVRFOXrr79WPD09laysrOLPr1mzRtHr9UpycrKiKIpSs2ZN5bXXXrttBkB5/fXXiz/OyspSdDqd8uuvv1bY+xRCVB3pARJCmIUHHniAOXPmlDjm5eVVfD8iIqLE5yIiIoiJiQHgxIkThIaG4uzsXPz5Tp06YTKZOHXqFDqdjosXL9KtW7c7ZmjZsmXxfWdnZ1xdXUlJSSnvWxJCaEgKICGEWXB2dr7pktTd6HQ6ABRFKb5/q8c4OjqW6ny2trY3PddkMpUpkxCiepAeICGERdizZ89NHzdu3BiApk2bEhMTQ3Z2dvHnd+7ciV6vp2HDhri6uhIcHMzvv/9epZmFENqRESAhhFnIy8sjOTm5xDEbGxu8vb0BWLZsGeHh4XTu3JmFCxeyb98+/ve//wEwatQo/vWvfzF27FjeeustLl++zAsvvMCYMWPw8/MD4K233uKZZ57B19eX3r17k5mZyc6dO3nhhReq9o0KIaqEFEBCCLPw22+/ERAQUOJYo0aNOHnyJKDO0Fq8eDHPPfcc/v7+LFy4kKZNmwLg5OTEunXrmDx5Mm3btsXJyYlHHnmEjz/+uPhcY8eOJTc3l08++YSXX34Zb29vHn300ap7g0KIKqVTFEXROoQQQtwLnU7HypUrGTRokNZRhBBmQnqAhBBCCGF1pAASQgghhNWRHiAhhNmTK/lCiLKSESAhhBBCWB0pgIQQQghhdaQAEkIIIYTVkQJICCGEEFZHCiAhhBBCWB0pgIQQQghhdaQAEkIIIYTVkQJICCGEEFZHCiAhhBBCWJ3/BxoLOapXFBKJAAAAAElFTkSuQmCC",
+      "text/plain": [
+       "<Figure size 640x480 with 1 Axes>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "# plot the learning curves\n",
+    "fig, ax = plt.subplots()\n",
+    "ax.plot(val_accs, label='val')\n",
+    "ax.plot(test_accs, label='test')\n",
+    "ax.set(xlabel='Epoch', ylabel='Accuracy')\n",
+    "ax.legend(loc='best');"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Best model checkpoint: nc_model/epoch-6\n"
+     ]
+    }
+   ],
+   "source": [
+    "# after training, the best model is saved to disk:\n",
+    "best_model_path = trainer.get_best_model_path()\n",
+    "print('Best model checkpoint:', best_model_path)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "total 1012\n",
+      "336 -rw-rw-r-- 1 ubuntu ubuntu 340551 Oct 11 15:07 model.bin\n",
+      "  4 drwxrw-rwx 2 ubuntu ubuntu   4096 Oct 11 15:07 movie\n",
+      "668 -rw-rw-r-- 1 ubuntu ubuntu 681259 Oct 11 15:07 optimizers.bin\n",
+      "  4 drwxrw-rwx 2 ubuntu ubuntu   4096 Oct 11 15:07 user\n"
+     ]
+    }
+   ],
+   "source": [
+    "# check the saved artifacts\n",
+    "!ls -ls {best_model_path}"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### 3. Inference with the trained model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/opt/conda/envs/pytorch/lib/python3.10/site-packages/torch/_utils.py:776: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()\n",
+      "  return self.fget.__get__(instance, owner)()\n"
+     ]
+    }
+   ],
+   "source": [
+    "# we can restore the model from the checkpoint:\n",
+    "model.restore_model(best_model_path)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Create a dataset for inference, we use the same MovieLens graph\n",
+    "infer_data = GSgnnNodeInferData(\n",
+    "    graph_name, \n",
+    "    constructed_graph_config,\n",
+    "    eval_ntypes=target_ntype,\n",
+    "    node_feat_field=node_feat_name,\n",
+    "    label_field=label_field)\n",
+    "\n",
+    "# Set up dataloader for the inference dataset\n",
+    "infer_dataloader = GSgnnNodeDataLoader(\n",
+    "    infer_data, infer_data.test_idxs, fanout=fanout, \n",
+    "    batch_size=100, device=device,\n",
+    "    train_task=False)\n",
+    "\n",
+    "\n",
+    "# Create an Inferrer object\n",
+    "infer = GSgnnNodePredictionInferrer(model)\n",
+    "infer.setup_device(device=device)\n",
+    "infer.setup_evaluator(evaluator)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Run inference on the inference dataset\n",
+    "infer.infer(infer_dataloader, \n",
+    "            save_embed_path=os.path.join(best_model_path, 'infer/embeddings'),\n",
+    "            save_prediction_path=os.path.join(best_model_path, 'infer/predictions'),\n",
+    "            use_mini_batch_infer=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "total 848K\n",
+      "-rw-rw-r-- 1 ubuntu ubuntu   40 Oct 11 15:07 emb_info.json\n",
+      "-rw-rw-r-- 1 ubuntu ubuntu 842K Oct 11 15:07 movie_emb.part00000.bin\n"
+     ]
+    }
+   ],
+   "source": [
+    "# The GNN embeddings on the inference graph are saved to:\n",
+    "!ls -lh {best_model_path}/infer/embeddings"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "total 16K\n",
+      "-rw-rw-r-- 1 ubuntu ubuntu 14K Oct 11 15:07 predict-00000.pt\n"
+     ]
+    }
+   ],
+   "source": [
+    "!ls -lh {best_model_path}/infer/predictions"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.11"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/python/graphstorm/eval/evaluator.py b/python/graphstorm/eval/evaluator.py
index 51b346c925..6127908afa 100644
--- a/python/graphstorm/eval/evaluator.py
+++ b/python/graphstorm/eval/evaluator.py
@@ -326,6 +326,11 @@ def best_iter_num(self):
         """
         return self._best_iter
 
+    @property
+    def history(self):
+        """ Evaluation history"""
+        return self._history
+
 class GSgnnRegressionEvaluator(GSgnnInstanceEvaluator):
     """ The class for user defined evaluator.
 

From 9f5218320fc49a2d30a6dab6475ed69810e5fbd7 Mon Sep 17 00:00:00 2001
From: Da Zheng <zhengda1936@gmail.com>
Date: Thu, 12 Oct 2023 08:02:10 -0700
Subject: [PATCH 6/6] Add the LM embedding cache (#523)

*Description of changes:*
Add a cache to save the LM embeddings on files to avoid expensive LM
computation when training the GNN model.

By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice.

---------

Co-authored-by: Ubuntu <ubuntu@ip-172-31-12-121.us-west-2.compute.internal>
Co-authored-by: Ubuntu <ubuntu@ip-172-31-0-17.us-west-2.compute.internal>
---
 examples/mag/mag_gnn_nc.yaml                  |   2 +
 python/graphstorm/config/argument.py          |  15 +
 python/graphstorm/gsf.py                      |   9 +-
 python/graphstorm/model/lm_embed.py           | 278 ++++++++++++------
 python/graphstorm/model/utils.py              |  84 ++++--
 .../end2end-tests/graphstorm-nc/mgpu_test.sh  |  11 +-
 tests/unit-tests/data_utils.py                |  26 ++
 tests/unit-tests/test_embed.py                |  87 +++++-
 8 files changed, 392 insertions(+), 120 deletions(-)

diff --git a/examples/mag/mag_gnn_nc.yaml b/examples/mag/mag_gnn_nc.yaml
index 58ff6dc333..e5c1183373 100644
--- a/examples/mag/mag_gnn_nc.yaml
+++ b/examples/mag/mag_gnn_nc.yaml
@@ -19,6 +19,8 @@ gsf:
     backend: gloo
     verbose: false
     save_perf_results_path: null
+  lm:
+    cache_lm_embed: true
   gnn:
     model_encoder_type: rgcn
     num_layers: 2
diff --git a/python/graphstorm/config/argument.py b/python/graphstorm/config/argument.py
index df43c56af2..771bc95fbb 100644
--- a/python/graphstorm/config/argument.py
+++ b/python/graphstorm/config/argument.py
@@ -646,6 +646,15 @@ def distill_lm_configs(self):
         # By default there is no distill_lm_config
         return None
 
+    @property
+    def cache_lm_embed(self):
+        """ Whether to cache the LM embeddings on files.
+        """
+        if hasattr(self, "_cache_lm_embed"):
+            return self._cache_lm_embed
+        else:
+            return None
+
     ###################### general gnn model related ######################
     @property
     def model_encoder_type(self):
@@ -2229,6 +2238,12 @@ def _add_lm_model_args(parser):
                  "to warmup a GNN model")
     group.add_argument("--max-seq-len", type=int, default=argparse.SUPPRESS,
                        help="The maximum of sequence length for distillation")
+    group.add_argument("--cache-lm-embed",
+            type=lambda x: (str(x).lower() in ['true', '1']),
+            default=argparse.SUPPRESS,
+            help="Whether to cache the LM embeddings in files. " + \
+                    "If the LM embeddings have been saved before, load the saved embeddings " + \
+                    "instead of computing the LM embeddings again.")
     return parser
 
 def _add_rgat_args(parser):
diff --git a/python/graphstorm/gsf.py b/python/graphstorm/gsf.py
index f9d0acf8a3..94bdacf774 100644
--- a/python/graphstorm/gsf.py
+++ b/python/graphstorm/gsf.py
@@ -518,18 +518,23 @@ def set_encoder(model, g, config, train_task):
     reconstruct_feats = len(config.construct_feat_ntype) > 0
     model_encoder_type = config.model_encoder_type
     if config.node_lm_configs is not None:
+        emb_path = os.path.join(os.path.dirname(config.part_config),
+                "cached_embs") if config.cache_lm_embed else None
         if model_encoder_type == "lm":
             # only use language model(s) as input layer encoder(s)
             encoder = GSPureLMNodeInputLayer(g, config.node_lm_configs,
                                              num_train=config.lm_train_nodes,
-                                             lm_infer_batch_size=config.lm_infer_batch_size)
+                                             lm_infer_batch_size=config.lm_infer_batch_size,
+                                             cached_embed_path=emb_path)
         else:
             encoder = GSLMNodeEncoderInputLayer(g, config.node_lm_configs,
                                                 feat_size, config.hidden_size,
                                                 num_train=config.lm_train_nodes,
                                                 lm_infer_batch_size=config.lm_infer_batch_size,
                                                 dropout=config.dropout,
-                                                use_node_embeddings=config.use_node_embeddings)
+                                                use_node_embeddings=config.use_node_embeddings,
+                                                cached_embed_path=emb_path,
+                                                force_no_embeddings=config.construct_feat_ntype)
     else:
         encoder = GSNodeEncoderInputLayer(g, feat_size, config.hidden_size,
                                           dropout=config.dropout,
diff --git a/python/graphstorm/model/lm_embed.py b/python/graphstorm/model/lm_embed.py
index 377f51e7b6..74ee02fc5c 100644
--- a/python/graphstorm/model/lm_embed.py
+++ b/python/graphstorm/model/lm_embed.py
@@ -19,7 +19,10 @@
 
 import logging
 import time
+import os
+import hashlib
 
+import numpy as np
 import torch as th
 from torch import nn
 import dgl
@@ -28,62 +31,8 @@
 from .embed import GSNodeEncoderInputLayer
 from .lm_model import init_lm_model
 from .lm_model import get_lm_node_feats
-from ..utils import get_rank, barrier, create_dist_tensor
-
-def update_bert_cache(g, lm_models, lm_emb_cache, lm_infer_batch_size, use_fp16=True):
-    """ Update the lm_emb_cache using lanaguage models.
-
-    Parameters
-    ----------
-    lm_models: LMModels
-        A collection of LM models and related information.
-    lm_emb_cache: dict
-        Language model embedding cache
-    lm_infer_batch_size: int
-        Language model inference batch size
-    use_fp16 : bool
-        Use float16 to store BERT embeddings.
-    """
-    for ntype in lm_models.ntypes:
-        start = time.time()
-        lm_model = lm_models.get_lm_model(ntype)
-        lm_node_feat = lm_models.get_lm_node_feat(ntype)
-        lm_model.eval()
-        hidden_size = lm_model.feat_size
-        # TODO we should not save the BERT embeddings on the graph data in the future.
-        if 'bert_emb' not in g.nodes[ntype].data:
-            g.nodes[ntype].data['bert_emb'] = create_dist_tensor(
-                        (g.number_of_nodes(ntype), hidden_size),
-                        name="bert_emb",
-                        dtype=th.float16 if use_fp16 else th.float32,
-                        part_policy=g.get_node_partition_policy(ntype),
-                        persistent=True)
-        input_emb = g.nodes[ntype].data['bert_emb']
-        infer_nodes = dgl.distributed.node_split(
-                th.ones((g.number_of_nodes(ntype),), dtype=th.bool),
-                partition_book=g.get_partition_book(),
-                ntype=ntype, force_even=False)
-        logging.debug("node %s, local infer set: %d, batch size: %d",
-                      ntype, len(infer_nodes), lm_infer_batch_size)
-
-        node_list = th.split(infer_nodes, lm_infer_batch_size)
-        input_ntypes = [ntype]
-        for input_nodes in node_list:
-            input_lm_feats = {}
-            input_lm_feats[ntype] = {
-                    fname: feat[input_nodes] for fname, feat in lm_node_feat.items()
-            }
-            text_embs = lm_model(input_ntypes, input_lm_feats)
-            if use_fp16:
-                input_emb[input_nodes] = text_embs[ntype].half().to('cpu')
-            else:
-                input_emb[input_nodes] = text_embs[ntype].to('cpu')
-        barrier()
-        if get_rank() == 0:
-            logging.info('Computing bert embedding on node %s takes %.3f seconds',
-                         ntype, time.time() - start)
-        lm_emb_cache[ntype] = input_emb
-        lm_model.train()
+from .utils import load_pytorch_embedding, save_embeddings
+from ..utils import get_rank, get_world_size, barrier, create_dist_tensor
 
 class LMModels(nn.Module):
     """ LM model collection
@@ -114,6 +63,7 @@ def __init__(self, g, node_lm_configs, num_train, lm_infer_batch_size):
         # to help find the right BERT model for a node type.
         self._lm_map = {}
         self._lm_models = nn.ModuleDict()
+        self._lm_model_names = {}
         self._lm_node_feats = {}
         for lm_config in node_lm_configs:
             lm_model = init_lm_model(lm_config,
@@ -121,6 +71,8 @@ def __init__(self, g, node_lm_configs, num_train, lm_infer_batch_size):
                                      lm_infer_batch_size=lm_infer_batch_size)
             # A list of node types sharing the same lm model
             lm_ntypes = lm_config["node_types"]
+            for ntype in lm_ntypes:
+                self._lm_model_names[ntype] = lm_config["model_name"]
             lm_node_feats = get_lm_node_feats(g, lm_model, lm_ntypes)
             for ntype, feats in lm_node_feats.items():
                 assert ntype not in self._lm_node_feats, \
@@ -153,7 +105,7 @@ def forward(self, input_nodes, lm_emb_cache=None):
             # No bert training, Get cached LM embedding
             # Note: self.lm_emb_cache is initialized by calling warmup
             for ntype, idx in input_nodes.items():
-                if ntype in lm_emb_cache:
+                if ntype in lm_emb_cache.ntypes:
                     lm_feats[ntype] = lm_emb_cache[ntype][idx].to(dev).float()
         else:
             # TODO: Release the bert cache properly
@@ -185,6 +137,36 @@ def get_lm_model(self, ntype):
         idx = self._lm_map[ntype]
         return self._lm_models[idx]
 
+    def get_lm_model_name(self, ntype):
+        """ Get the LM model name on a node type.
+
+        Parameters
+        ----------
+        ntype : str
+            The node type
+
+        Returns
+        -------
+        str : the model name
+        """
+        return self._lm_model_names[ntype]
+
+    def get_lm_model_hash(self, ntype):
+        """ Compute the hash code of a LM model on a given node type.
+
+        Parameters
+        ----------
+        ntype : str
+            The node type
+
+        Returns
+        -------
+        str : the hash code of the model.
+        """
+        weights = [th.flatten(param.data) for param in self.get_lm_model(ntype).parameters()]
+        weights = th.cat(weights).cpu().numpy()
+        return hashlib.sha1(weights.view(np.uint8)).hexdigest()
+
     def get_lm_node_feat(self, ntype):
         """ Get the LM node features.
 
@@ -233,6 +215,142 @@ def lm_models(self):
         """
         return list(self._lm_models.values())
 
+class LMCache:
+    """ Cache for the LM embeddings.
+
+    Parameters
+    ----------
+    lm_models: LMModels
+        A collection of LM models and related information.
+    embed_path : str
+        The path where the embedding files are stored.
+    """
+    def __init__(self, g, lm_models, embed_path=None):
+        self._g = g
+        self._lm_models = lm_models
+        self._lm_emb_cache = {}
+        self._embed_path = embed_path
+
+    def _get_model_hash(self, ntype):
+        """ Get the hash code of a model.
+
+        If necessary, we may cache the hash code in the future.
+        """
+        return self._lm_models.get_lm_model_hash(ntype)
+
+    def _get_model_name(self, ntype):
+        """ Get the model name
+
+        The model name should be the original model name followed with a hash code:
+        "model_name"-"hash_code"
+
+        """
+        model_name = self._lm_models.get_lm_model_name(ntype)
+        assert "/" not in model_name, \
+                f"We only support builtin LM models for now. The model name is {model_name}."
+        model_hash = self._get_model_hash(ntype)
+        # We only take the first 10 characters of the hash code to construct the model name.
+        return model_name + "-" + model_hash[0:10]
+
+    def _load_embeddings(self):
+        """ Load LM embeddings from files.
+        """
+        for ntype in self._lm_models.ntypes:
+            embed_path = os.path.join(os.path.join(self._embed_path, ntype),
+                    self._get_model_name(ntype))
+            if os.path.exists(embed_path):
+                if get_rank() == 0:
+                    logging.info("load LM embedding from %s for node type %s",
+                            embed_path, ntype)
+                self._lm_emb_cache[ntype] = load_pytorch_embedding(embed_path,
+                        self._g.get_node_partition_policy(ntype), "bert_emb")
+
+    def _save_embeddings(self):
+        """ Save LM embeddings.
+        """
+        for ntype in self._lm_models.ntypes:
+            embed_path = os.path.join(os.path.join(self._embed_path, ntype),
+                    self._get_model_name(ntype))
+            save_embeddings(embed_path, self._lm_emb_cache[ntype], get_rank(), get_world_size())
+
+    def __len__(self):
+        return len(self._lm_emb_cache)
+
+    def __getitem__(self, ntype):
+        """ Get the cached embedding of a node type.
+        """
+        return self._lm_emb_cache[ntype]
+
+    @property
+    def ntypes(self):
+        """ Get the node types with embedding cache.
+        """
+        return self._lm_models.ntypes
+
+    def update_cache(self, lm_infer_batch_size, use_fp16=True):
+        """ Update the LM embedding cache.
+
+        Parameters
+        ----------
+        lm_infer_batch_size: int
+            Language model inference batch size
+        use_fp16 : bool
+            Use float16 to store BERT embeddings.
+        """
+        # If the embeddings have been cached, we just load them instead of
+        # computing them from scratch.
+        if self._embed_path is not None:
+            self._load_embeddings()
+
+        # If all embeddings are cached, don't compute the embeddings again.
+        if np.all([ntype in self._lm_emb_cache for ntype in self._lm_models.ntypes]):
+            return
+
+        for ntype in self._lm_models.ntypes:
+            if get_rank() == 0:
+                logging.debug("compute embedding for node type %s", ntype)
+            start = time.time()
+            lm_model = self._lm_models.get_lm_model(ntype)
+            lm_node_feat = self._lm_models.get_lm_node_feat(ntype)
+            lm_model.eval()
+            hidden_size = lm_model.feat_size
+            if ntype not in self._lm_emb_cache:
+                self._lm_emb_cache[ntype] = create_dist_tensor(
+                        (self._g.number_of_nodes(ntype), hidden_size),
+                        name="bert_emb",
+                        dtype=th.float16 if use_fp16 else th.float32,
+                        part_policy=self._g.get_node_partition_policy(ntype),
+                        persistent=True)
+            emb = self._lm_emb_cache[ntype]
+            infer_nodes = dgl.distributed.node_split(
+                    th.ones((self._g.number_of_nodes(ntype),), dtype=th.bool),
+                    partition_book=self._g.get_partition_book(),
+                    ntype=ntype, force_even=False)
+            logging.debug("node %s, local infer set: %d, batch size: %d",
+                          ntype, len(infer_nodes), lm_infer_batch_size)
+
+            node_list = th.split(infer_nodes, lm_infer_batch_size)
+            input_ntypes = [ntype]
+            with th.no_grad():
+                for input_nodes in node_list:
+                    input_lm_feats = {}
+                    input_lm_feats[ntype] = {
+                            fname: feat[input_nodes] for fname, feat in lm_node_feat.items()
+                    }
+                    text_embs = lm_model(input_ntypes, input_lm_feats)
+                    if use_fp16:
+                        emb[input_nodes] = text_embs[ntype].half().to('cpu')
+                    else:
+                        emb[input_nodes] = text_embs[ntype].to('cpu')
+            barrier()
+            if get_rank() == 0:
+                logging.info('Computing bert embedding on node %s takes %.3f seconds',
+                             ntype, time.time() - start)
+            lm_model.train()
+
+        if self._embed_path is not None:
+            self._save_embeddings()
+
 class GSPureLMNodeInputLayer(GSNodeInputLayer):
     """The input embedding layer with language model only for all nodes in a
     heterogeneous graph.
@@ -255,7 +373,9 @@ class GSPureLMNodeInputLayer(GSNodeInputLayer):
     lm_infer_batch_size: int
         Batch size used for computing text embeddings for static lm model. Default: 16
     use_fp16 : bool 
-        Use float16 to store BERT embeddings. Default: True
+        Use float16 to store LM embeddings. Default: True
+    cached_embed_path : str
+        The path where the LM embeddings are cached.
     
     Examples:
     ----------
@@ -285,7 +405,8 @@ def __init__(self,
                  node_lm_configs,
                  num_train=0,
                  lm_infer_batch_size=16,
-                 use_fp16=True):
+                 use_fp16=True,
+                 cached_embed_path=None):
         super(GSPureLMNodeInputLayer, self).__init__(g)
         assert node_lm_configs is not None and len(node_lm_configs) > 0, \
             "language model configurations must be provided"
@@ -295,7 +416,7 @@ def __init__(self,
         self.lm_infer_batch_size = lm_infer_batch_size
         self.use_fp16 = use_fp16
         self.use_cache = False
-        self.lm_emb_cache = {}
+        self.lm_emb_cache = LMCache(g, self._lm_models, embed_path=cached_embed_path)
 
         self._feat_size = self._lm_models.feat_size
         for lm_model in self._lm_models.lm_models:
@@ -329,13 +450,8 @@ def prepare(self, g):
         if self.num_train == 0:
             self.freeze(g)
 
-    def freeze(self, g):
+    def freeze(self, _):
         """ Generate Bert caching if needed
-
-        Parameters
-        ----------
-        g : DistGraph
-            The distributed graph object.
         """
         # The lm_emb_cache is used in following cases:
         # 1) We don't need to fine-tune Bert, i.e., train_nodes == 0.
@@ -347,11 +463,7 @@ def freeze(self, g):
         #    nodes are set to 0 and the lm_emb_cache is not refreshed.
         #
         # 3) if train_nodes > 0, no emb_cache is used unless Case 2.
-        update_bert_cache(g,
-                          self._lm_models,
-                          self.lm_emb_cache,
-                          self.lm_infer_batch_size,
-                          use_fp16=self.use_fp16)
+        self.lm_emb_cache.update_cache(self.lm_infer_batch_size, use_fp16=self.use_fp16)
         self.use_cache = True
 
     def require_cache_embed(self):
@@ -431,6 +543,8 @@ class GSLMNodeEncoderInputLayer(GSNodeEncoderInputLayer):
         available. Default: False
     use_fp16 : bool
         Use float16 to store the BERT embeddings. Default: True
+    cached_embed_path : str
+        The path where the LM embeddings are cached.
 
     Examples:
     ----------
@@ -468,7 +582,9 @@ def __init__(self,
                  activation=None,
                  dropout=0.0,
                  use_node_embeddings=False,
-                 use_fp16=True):
+                 use_fp16=True,
+                 cached_embed_path=None,
+                 force_no_embeddings=None):
         assert node_lm_configs is not None and len(node_lm_configs) > 0, \
             "language model configurations must be provided"
 
@@ -489,11 +605,12 @@ def __init__(self,
         self.use_fp16 = use_fp16
         self.lm_infer_batch_size = lm_infer_batch_size
         self.use_cache = False
-        self.lm_emb_cache = {}
+        self.lm_emb_cache = LMCache(g, lm_models, embed_path=cached_embed_path)
 
         super(GSLMNodeEncoderInputLayer, self).__init__(
             g, adjust_feat_size, embed_size,
-            activation, dropout, use_node_embeddings)
+            activation, dropout, use_node_embeddings,
+            force_no_embeddings=force_no_embeddings)
         self._lm_models = lm_models
 
     def get_general_dense_parameters(self):
@@ -522,13 +639,8 @@ def prepare(self, g):
         if self.num_train == 0:
             self.freeze(g)
 
-    def freeze(self, g):
+    def freeze(self, _):
         """ Generate Bert caching if needed
-
-        Parameters
-        ----------
-        g : DistGraph
-            The distributed graph object.
         """
         # The lm_emb_cache is used in following cases:
         # 1) We don't need to fine-tune Bert, i.e., train_nodes == 0.
@@ -540,11 +652,7 @@ def freeze(self, g):
         #    nodes are set to 0 and the lm_emb_cache is not refreshed.
         #
         # 3) if train_nodes > 0, no emb_cache is used unless Case 2.
-        update_bert_cache(g,
-                          self._lm_models,
-                          self.lm_emb_cache,
-                          self.lm_infer_batch_size,
-                          use_fp16=self.use_fp16)
+        self.lm_emb_cache.update_cache(self.lm_infer_batch_size, use_fp16=self.use_fp16)
         self.use_cache = True
 
     def unfreeze(self):
diff --git a/python/graphstorm/model/utils.py b/python/graphstorm/model/utils.py
index 304eb4fa90..c879c8c712 100644
--- a/python/graphstorm/model/utils.py
+++ b/python/graphstorm/model/utils.py
@@ -28,7 +28,7 @@
 import dgl
 
 from ..gconstruct.file_io import stream_dist_tensors_to_hdf5
-from ..utils import get_rank, barrier, get_world_size
+from ..utils import get_rank, barrier, get_world_size, create_dist_tensor
 from ..data.utils import alltoallv_cpu, alltoallv_nccl
 
 def pad_file_index(file_index, width=5):
@@ -467,19 +467,45 @@ def remap_embeddings(embeddings, rank, world_size,
                 start, end = _get_data_range(rank, world_size, len(emb))
                 # we need to keep emb to be dist tensor unchanged
                 emb[th.arange(start, end)] = emb[nid_mapping[name]]
-            th.distributed.barrier()
+            barrier()
 
     return embeddings
 
-def save_pytorch_embeddings(model_path, embeddings, rank, world_size,
+def load_pytorch_embedding(emb_path, part_policy, name):
+    """ Load embedding tensor in Pytorch format.
+
+    Parameters
+    ----------
+    emb_path : str
+        The path of the save embedding files.
+    part_policy : dgl.distributed.PartitionPolicy
+        The partitioning policy
+    name : str
+        The name of the created distributed tensor.
+
+    Returns
+    -------
+    DistTensor : the loaded embeddings.
+    """
+    rank = get_rank()
+    world_size = get_world_size()
+    emb = th.load(os.path.join(emb_path, f'emb.part{pad_file_index(rank)}.bin'))
+    dist_emb = create_dist_tensor((part_policy.get_size(), emb.shape[1]), emb.dtype,
+            name=name, part_policy=part_policy)
+    start, end = _get_data_range(rank, world_size, len(dist_emb))
+    dist_emb[start:end] = emb
+    barrier()
+    return dist_emb
+
+def save_pytorch_embeddings(emb_path, embeddings, rank, world_size,
     device=th.device('cpu'), node_id_mapping_file=None):
     """ Save embeddings through pytorch a distributed way
 
         Parameters
         ----------
-        model_path : str
-            The path of the folder where the model is saved.
-        embeddings : DistTensor
+        emb_path : str
+            The path of the folder where the embeddings are saved.
+        embeddings : DistTensor or dict of DistTensor
             Embeddings to save
         rank : int
             Rank of the current process in a distributed environment.
@@ -499,9 +525,9 @@ def save_pytorch_embeddings(model_path, embeddings, rank, world_size,
         #     - owner of the folder can read, write, and execute;
         #     - owner' group can read, write;
         #     - others can read, write, and execute.
-        os.chmod(model_path, 0o767)
+        os.chmod(emb_path, 0o767)
 
-    # make sure the model_path permission is changed before other process start to save
+    # make sure the emb_path permission is changed before other process start to save
     barrier()
 
     assert rank < world_size
@@ -540,24 +566,24 @@ def save_pytorch_embeddings(model_path, embeddings, rank, world_size,
     if isinstance(embeddings, dict):
         # embedding per node type
         for name, emb in embeddings.items():
-            th.save(emb, os.path.join(model_path, f'{name}_emb.part{pad_file_index(rank)}.bin'))
+            th.save(emb, os.path.join(emb_path, f'{name}_emb.part{pad_file_index(rank)}.bin'))
             emb_info["emb_name"].append(name)
     else:
-        th.save(embeddings, os.path.join(model_path, f'emb.part{pad_file_index(rank)}.bin'))
+        th.save(embeddings, os.path.join(emb_path, f'emb.part{pad_file_index(rank)}.bin'))
         emb_info["emb_name"] = None
 
     if rank == 0:
-        with open(os.path.join(model_path, "emb_info.json"), 'w', encoding='utf-8') as f:
+        with open(os.path.join(emb_path, "emb_info.json"), 'w', encoding='utf-8') as f:
             f.write(json.dumps(emb_info))
 
-def save_hdf5_embeddings(model_path, embeddings, rank, world_size,
+def save_hdf5_embeddings(emb_path, embeddings, rank, world_size,
     device=th.device('cpu'), node_id_mapping_file=None):
     """ Save embeddings through hdf5 into a single file.
 
         Parameters
         ----------
-        model_path : str
-            The path of the folder where the model is saved.
+        emb_path : str
+            The path of the folder where the embeddings are saved.
         embeddings : DistTensor
             Embeddings to save
         rank : int
@@ -572,21 +598,20 @@ def save_hdf5_embeddings(model_path, embeddings, rank, world_size,
             Path to the file storing node id mapping generated by the
             graph partition algorithm.
     """
-    mapped_embeds = remap_embeddings(embeddings, rank, world_size,
-    node_id_mapping_file, device)
+    mapped_embeds = remap_embeddings(embeddings, rank, world_size, node_id_mapping_file, device)
     if rank == 0:
-        stream_dist_tensors_to_hdf5(mapped_embeds, os.path.join(model_path, "embed_dict.hdf5"))
+        stream_dist_tensors_to_hdf5(mapped_embeds, os.path.join(emb_path, "embed_dict.hdf5"))
 
-def save_embeddings(model_path, embeddings, rank, world_size,
+def save_embeddings(emb_path, embeddings, rank, world_size,
     device=th.device('cpu'), node_id_mapping_file=None,
     save_embed_format="pytorch"):
     """ Save embeddings.
 
         Parameters
         ----------
-        model_path : str
-            The path of the folder where the model is saved.
-        embeddings : DistTensor
+        emb_path : str
+            The path of the folder where the embeddings are saved.
+        embeddings : DistTensor or dict of DistTensor
             Embeddings to save
         rank : int
             Rank of the current process in a distributed environment.
@@ -603,22 +628,21 @@ def save_embeddings(model_path, embeddings, rank, world_size,
             The format of saved embeddings.
             Currently support ["pytorch", "hdf5"].
     """
-    os.makedirs(model_path, exist_ok=True)
+    os.makedirs(emb_path, exist_ok=True)
     if save_embed_format == "pytorch":
         if rank == 0:
-            logging.info("Writing GNN embeddings to "\
-                "%s in pytorch format.", model_path)
-        save_pytorch_embeddings(model_path, embeddings, rank, world_size,
+            logging.info("Writing GNN embeddings to %s in pytorch format.",
+                    emb_path)
+        save_pytorch_embeddings(emb_path, embeddings, rank, world_size,
             device, node_id_mapping_file)
     elif save_embed_format == "hdf5":
         if rank == 0:
-            logging.info("Writing GNN embeddings to "\
-                "%s in hdf5 format.", \
-                os.path.join(model_path, 'embed_dict.hdf5'))
-        save_hdf5_embeddings(model_path, embeddings, rank, world_size,
+            logging.info("Writing GNN embeddings to %s in hdf5 format.", \
+                os.path.join(emb_path, 'embed_dict.hdf5'))
+        save_hdf5_embeddings(emb_path, embeddings, rank, world_size,
             device, node_id_mapping_file)
     else:
-        raise ValueError(f"{model_path} is not supported for save_embed_format")
+        raise ValueError(f"{emb_path} is not supported for save_embed_format")
 
 def shuffle_predict(predictions, id_mapping_file, pred_type,
                     rank, world_size, device):
diff --git a/tests/end2end-tests/graphstorm-nc/mgpu_test.sh b/tests/end2end-tests/graphstorm-nc/mgpu_test.sh
index c77b4e0af6..7a532d257f 100644
--- a/tests/end2end-tests/graphstorm-nc/mgpu_test.sh
+++ b/tests/end2end-tests/graphstorm-nc/mgpu_test.sh
@@ -177,7 +177,7 @@ fi
 rm -fr /data/gsgnn_nc_ml/*
 
 echo "**************dataset: MovieLens classification, RGCN layer: 1, node feat: BERT nodes: movie, user inference: mini-batch save model save emb node"
-python3 -m graphstorm.run.gs_node_classification --workspace $GS_HOME/training_scripts/gsgnn_np/ --num-trainers $NUM_TRAINERS --num-servers 1 --num-samplers 0 --part-config /data/movielen_100k_lm_encoder_train_val_1p_4t/movie-lens-100k-text.json --ip-config ip_list.txt --ssh-port 2222 --cf ml_nc_utext.yaml  --save-model-path /data/gsgnn_nc_ml_text/ --topk-model-to-save 1 --save-embed-path /data/gsgnn_nc_ml_text/emb/ --num-epochs 3 --logging-file /tmp/train_log.txt
+python3 -m graphstorm.run.gs_node_classification --workspace $GS_HOME/training_scripts/gsgnn_np/ --num-trainers $NUM_TRAINERS --num-servers 1 --num-samplers 0 --part-config /data/movielen_100k_lm_encoder_train_val_1p_4t/movie-lens-100k-text.json --ip-config ip_list.txt --ssh-port 2222 --cf ml_nc_utext.yaml  --save-model-path /data/gsgnn_nc_ml_text/ --topk-model-to-save 1 --save-embed-path /data/gsgnn_nc_ml_text/emb/ --num-epochs 3 --logging-file /tmp/train_log.txt --cache-lm-embed true
 
 error_and_exit $?
 
@@ -193,6 +193,15 @@ echo "The best model is saved in epoch $best_epoch"
 
 rm /tmp/train_log.txt
 
+echo "*************use cached LM embeddings"
+# Run the model training again and this time it should load the BERT embeddings saved
+# in the previous run.
+python3 -m graphstorm.run.gs_node_classification --workspace $GS_HOME/training_scripts/gsgnn_np/ --num-trainers $NUM_TRAINERS --num-servers 1 --num-samplers 0 --part-config /data/movielen_100k_lm_encoder_train_val_1p_4t/movie-lens-100k-text.json --ip-config ip_list.txt --ssh-port 2222 --cf ml_nc_utext.yaml --num-epochs 3 --logging-file /tmp/train_log.txt --cache-lm-embed true
+
+error_and_exit $?
+
+rm /tmp/train_log.txt
+
 echo "**************dataset: Movielens, do inference on saved model, decoder: dot"
 python3 -m graphstorm.run.gs_node_classification --inference --workspace $GS_HOME/inference_scripts/np_infer/ --num-trainers $NUM_INFERs --num-servers 1 --num-samplers 0 --part-config /data/movielen_100k_lm_encoder_train_val_1p_4t/movie-lens-100k-text.json --ip-config ip_list.txt --ssh-port 2222 --cf ml_nc_text_infer.yaml --use-mini-batch-infer false   --save-embed-path /data/gsgnn_nc_ml_text/infer-emb/ --restore-model-path /data/gsgnn_nc_ml_text/epoch-$best_epoch/ --save-prediction-path /data/gsgnn_nc_ml_text/prediction/ --logging-file /tmp/log.txt --logging-level debug
 
diff --git a/tests/unit-tests/data_utils.py b/tests/unit-tests/data_utils.py
index 6f47295de7..35a16047d5 100644
--- a/tests/unit-tests/data_utils.py
+++ b/tests/unit-tests/data_utils.py
@@ -17,6 +17,7 @@
 """
 
 import os
+import json
 import dgl
 import numpy as np
 import torch as th
@@ -421,6 +422,31 @@ def generate_dummy_dist_graph_multi_target_ntypes(dirname, size='tiny', graph_na
     return partion_and_load_distributed_graph(hetero_graph=hetero_graph, dirname=dirname,
                                               graph_name=graph_name)
 
+def load_lm_graph(part_config):
+    with open(part_config) as f:
+        part_metadata = json.load(f)
+    g = dgl.distributed.DistGraph(graph_name=part_metadata["graph_name"],
+            part_config=part_config)
+
+    bert_model_name = "bert-base-uncased"
+    max_seq_length = 8
+    lm_config = [{"lm_type": "bert",
+                  "model_name": bert_model_name,
+                  "gradient_checkpoint": True,
+                  "node_types": ["n0"]}]
+    feat_size = get_feat_size(g, {'n0' : ['feat']})
+    input_text = ["Hello world!"]
+    tokenizer = AutoTokenizer.from_pretrained(bert_model_name)
+    input_ids, valid_len, attention_mask, _ = \
+        create_tokens(tokenizer=tokenizer,
+                      input_text=input_text,
+                      max_seq_length=max_seq_length,
+                      num_node=g.number_of_nodes('n0'))
+
+    g.nodes['n0'].data[TOKEN_IDX] = input_ids
+    g.nodes['n0'].data[VALID_LEN] = valid_len
+    return g, lm_config
+
 def create_lm_graph(tmpdirname):
     """ Create a graph with textual feaures
         Only n0 has a textual feature.
diff --git a/tests/unit-tests/test_embed.py b/tests/unit-tests/test_embed.py
index 4b49e17165..49d1ab25a8 100644
--- a/tests/unit-tests/test_embed.py
+++ b/tests/unit-tests/test_embed.py
@@ -14,6 +14,7 @@
     limitations under the License.
 """
 
+import multiprocessing as mp
 import pytest
 import torch as th
 from torch import nn
@@ -25,15 +26,16 @@
 
 import dgl
 from transformers import AutoTokenizer
+import graphstorm as gs
 from graphstorm import get_feat_size
 from graphstorm.model import GSNodeEncoderInputLayer, GSLMNodeEncoderInputLayer, GSPureLMNodeInputLayer
 from graphstorm.model.embed import compute_node_input_embeddings
 from graphstorm.dataloading.dataset import prepare_batch_input
 from graphstorm.model.lm_model import TOKEN_IDX, ATT_MASK_IDX, VALID_LEN
-
+from graphstorm.model.lm_embed import LMModels, LMCache
 
 from data_utils import generate_dummy_dist_graph
-from data_utils import create_lm_graph, create_lm_graph2
+from data_utils import create_lm_graph, create_lm_graph2, load_lm_graph
 from util import create_tokens
 
 # In this case, we only use the node features to generate node embeddings.
@@ -202,6 +204,85 @@ def test_compute_embed(dev):
     th.distributed.destroy_process_group()
     dgl.distributed.kvstore.close_kvstore()
 
+def test_lm_cache():
+    # initialize the torch distributed environment
+    th.distributed.init_process_group(backend='gloo',
+                                      init_method='tcp://127.0.0.1:23456',
+                                      rank=0,
+                                      world_size=1)
+    with tempfile.TemporaryDirectory() as tmpdirname:
+        lm_config, feat_size, input_ids, attention_mask, g, _ = \
+            create_lm_graph(tmpdirname)
+
+        lm_models = LMModels(g, lm_config, 0, 10)
+        lm_cache = LMCache(g, lm_models, tmpdirname)
+        lm_cache.update_cache(100)
+        assert len(lm_cache) == 1
+        assert len(lm_cache.ntypes) == 1
+        assert lm_cache.ntypes[0] == 'n0'
+
+        # Create the second cache. It should loads the embeddings from
+        # the first cache.
+        lm_cache2 = LMCache(g, lm_models, tmpdirname)
+        lm_cache2.update_cache(100)
+        assert len(lm_cache2) == 1
+        emb1 = lm_cache["n0"]
+        emb2 = lm_cache2["n0"]
+        assert np.all(emb1[0:len(emb1)].numpy() == emb2[0:len(emb2)].numpy())
+
+        # If the model is changed, the model name should also be changed.
+        model_name = lm_cache._get_model_name("n0")
+        for param in lm_models.get_lm_model("n0").parameters():
+            param.data += 1
+        model_name1 = lm_cache._get_model_name("n0")
+        assert model_name != model_name1
+
+    th.distributed.destroy_process_group()
+    dgl.distributed.kvstore.close_kvstore()
+
+def run_dist_cache(part_config, tmpdirname):
+    gs.initialize(ip_config=None, backend="gloo")
+    g, lm_config = load_lm_graph(part_config)
+    lm_models = LMModels(g, lm_config, 0, 10)
+    lm_cache = LMCache(g, lm_models, tmpdirname)
+    lm_cache.update_cache(100)
+    assert len(lm_cache) == 1
+    assert len(lm_cache.ntypes) == 1
+    assert lm_cache.ntypes[0] == 'n0'
+
+    # Create the second cache. It should loads the embeddings from
+    # the first cache.
+    lm_cache2 = LMCache(g, lm_models, tmpdirname)
+    lm_cache2.update_cache(100)
+    assert len(lm_cache2) == 1
+    emb1 = lm_cache["n0"]
+    emb2 = lm_cache2["n0"]
+    assert np.all(emb1[0:len(emb1)].numpy() == emb2[0:len(emb2)].numpy())
+
+def test_mp_lm_cache():
+    # initialize the torch distributed environment
+    th.distributed.init_process_group(backend='gloo',
+                                      init_method='tcp://127.0.0.1:23456',
+                                      rank=0,
+                                      world_size=1)
+    with tempfile.TemporaryDirectory() as tmpdirname:
+        lm_config, feat_size, input_ids, attention_mask, _, part_config = \
+            create_lm_graph(tmpdirname)
+
+        ctx = mp.get_context('spawn')
+        p0 = ctx.Process(target=run_dist_cache, args=(part_config, tmpdirname))
+        p1 = ctx.Process(target=run_dist_cache, args=(part_config, tmpdirname))
+
+        p0.start()
+        p1.start()
+        p0.join()
+        p1.join()
+        assert p0.exitcode == 0
+        assert p1.exitcode == 0
+
+    th.distributed.destroy_process_group()
+    dgl.distributed.kvstore.close_kvstore()
+
 def test_lm_infer():
     # initialize the torch distributed environment
     th.distributed.init_process_group(backend='gloo',
@@ -424,6 +505,8 @@ def rand_init(m):
 
 
 if __name__ == '__main__':
+    test_lm_cache()
+    test_mp_lm_cache()
     test_input_layer1(None)
     test_input_layer1(F.relu)
     test_input_layer2()