[Refactor] Refactor Evaluator

docs/source/advanced/own-models.rst

@@ -263,13 +263,13 @@ The GraphStorm trainers can have evaluators and task trackers associated. The fo
 .. code-block:: python
 
     # Optional: set up a evaluator
-    evaluator = GSgnnAccEvaluator(config.eval_frequency,
-                                  config.eval_metric,
-                                  config.multilabel,
-                                  config.use_early_stop,
-                                  config.early_stop_burnin_rounds,
-                                  config.early_stop_rounds,
-                                  config.early_stop_strategy)
+    evaluator = GSgnnClassificationEvaluator(config.eval_frequency,
+                                             config.eval_metric,
+                                             config.multilabel,
+                                             config.use_early_stop,
+                                             config.early_stop_burnin_rounds,
+                                             config.early_stop_rounds,
+                                             config.early_stop_strategy)
     trainer.setup_evaluator(evaluator)
     # Optional: set up a task tracker to show the progress of training.
     tracker = GSSageMakerTaskTracker(config.eval_frequency)

docs/source/api/graphstorm.eval.rst

@@ -7,8 +7,10 @@ graphstorm.eval
     Learning (GML) tasks.
 
     If users want to implement customized evaluators or evaluation methods, a best practice is to
-    extend base evaluators, i.e., the ``GSgnnInstanceEvaluator`` class for node or edge prediction
-    tasks, and ``GSgnnLPEvaluator`` for link prediction tasks, and then implement the abstract methods.
+    extend the base evaluator, i.e., the ``GSgnnBaseEvaluator``, and the corresponding evaluation
+    interfaces, e.g., ``GSgnnPredictionEvalInterface``` for prediction evaluation, and
+    ``GSgnnLPRankingEvalInterface`` for ranking based link prediction evaluation, and then
+    implement the abstract methods defined in those interface classes.
 
 .. currentmodule:: graphstorm.eval
 
@@ -20,8 +22,9 @@ Base Evaluators
     :nosignatures:
     :template: evaltemplate.rst
 
-    GSgnnInstanceEvaluator
-    GSgnnLPEvaluator
+    GSgnnBaseEvaluator
+    GSgnnPredictionEvalInterface
+    GSgnnLPRankingEvalInterface
 
 Evaluators
 -----------
@@ -31,8 +34,7 @@ Evaluators
     :nosignatures:
     :template: evaltemplate.rst
 
-    GSgnnLPEvaluator
+    GSgnnClassificationEvaluator
+    GSgnnRegressionEvaluator
     GSgnnMrrLPEvaluator
     GSgnnPerEtypeMrrLPEvaluator
-    GSgnnAccEvaluator
-    GSgnnRegressionEvaluator

examples/customized_models/HGT/hgt_nc.py

@@ -14,7 +14,7 @@
 from graphstorm.inference import GSgnnNodePredictionInferrer
 from graphstorm.dataloading import GSgnnNodeTrainData, GSgnnNodeInferData
 from graphstorm.dataloading import GSgnnNodeDataLoader
-from graphstorm.eval import GSgnnAccEvaluator
+from graphstorm.eval import GSgnnClassificationEvaluator
 from graphstorm.tracker import GSSageMakerTaskTracker
 from graphstorm.utils import get_device
 
@@ -326,13 +326,13 @@ def main(args):
                                           train_task=False)
 
     # Optional: set up a evaluator
-    evaluator = GSgnnAccEvaluator(config.eval_frequency,
-                                  config.eval_metric,
-                                  config.multilabel,
-                                  config.use_early_stop,
-                                  config.early_stop_burnin_rounds,
-                                  config.early_stop_rounds,
-                                  config.early_stop_strategy)
+    evaluator = GSgnnClassificationEvaluator(config.eval_frequency,
+                                             config.eval_metric,
+                                             config.multilabel,
+                                             config.use_early_stop,
+                                             config.early_stop_burnin_rounds,
+                                             config.early_stop_rounds,
+                                             config.early_stop_strategy)
     trainer.setup_evaluator(evaluator)
     # Optional: set up a task tracker to show the progress of training.
     tracker = GSSageMakerTaskTracker(config.eval_frequency)

examples/peft_llm_gnn/main_lp.py

@@ -54,14 +54,12 @@ def main(config_args):
     trainer.setup_device(device=get_device())
 
     # set evaluator
-    evaluator = GSgnnMrrLPEvaluator(config.eval_frequency,
-        train_data,
-        config.num_negative_edges_eval,
-        config.lp_decoder_type,
-        config.use_early_stop,
-        config.early_stop_burnin_rounds,
-        config.early_stop_rounds,
-        config.early_stop_strategy
+    evaluator = GSgnnMrrLPEvaluator(
+        eval_frequency=config.eval_frequency,
+        use_early_stop=config.use_early_stop,
+        early_stop_burnin_rounds=config.early_stop_burnin_rounds,
+        early_stop_rounds=config.early_stop_rounds,
+        early_stop_strategy=config.early_stop_strategy
     )
     # disbale validation for efficiency
     # trainer.setup_evaluator(evaluator)

examples/peft_llm_gnn/main_nc.py

@@ -3,7 +3,7 @@
 from graphstorm.config import get_argument_parser
 from graphstorm.config import GSConfig
 from graphstorm.dataloading import GSgnnNodeDataLoader
-from graphstorm.eval import GSgnnAccEvaluator
+from graphstorm.eval import GSgnnClassificationEvaluator
 from graphstorm.dataloading import GSgnnNodeTrainData
 from graphstorm.utils import get_device
 from graphstorm.inference import GSgnnNodePredictionInferrer
@@ -52,7 +52,7 @@ def main(config_args):
     trainer.setup_device(device=get_device())
 
     # set evaluator
-    evaluator = GSgnnAccEvaluator(
+    evaluator = GSgnnClassificationEvaluator(
         config.eval_frequency,
         config.eval_metric,
         config.multilabel,

examples/standalone_mode_demo.ipynb

@@ -39,7 +39,7 @@
     "from graphstorm.dataloading import GSgnnNodeTrainData, GSgnnNodeDataLoader, GSgnnNodeInferData\n",
     "from graphstorm.model import GSgnnNodeModel, GSNodeEncoderInputLayer, EntityClassifier, ClassifyLossFunc, RelationalGCNEncoder\n",
     "from graphstorm.inference import GSgnnNodePredictionInferrer\n",
-    "from graphstorm.eval import GSgnnAccEvaluator"
+    "from graphstorm.eval import GSgnnClassificationEvaluator"
    ]
   },
   {
@@ -315,9 +315,8 @@
     "trainer.setup_device(device=device)\n",
     "\n",
     "# set up evaluator for the trainer:\n",
-    "evaluator = GSgnnAccEvaluator(\n",
+    "evaluator = GSgnnClassificationEvaluator(\n",
     "    eval_frequency=10000,\n",
-    "    eval_metric=['accuracy'],\n",
     "    multilabel=multilabel)\n",
     "\n",
     "trainer.setup_evaluator(evaluator)"

examples/temporal_graph_learning/main_nc.py

@@ -3,7 +3,7 @@
 from graphstorm.config import get_argument_parser
 from graphstorm.config import GSConfig
 from graphstorm.dataloading import GSgnnNodeDataLoader
-from graphstorm.eval import GSgnnAccEvaluator
+from graphstorm.eval import GSgnnClassificationEvaluator
 from graphstorm.dataloading import GSgnnNodeTrainData
 from graphstorm.utils import get_device
 from graphstorm.trainer import GSgnnNodePredictionTrainer
@@ -45,7 +45,7 @@ def main(config_args):
     trainer.setup_device(device=get_device())
 
     # set evaluator
-    evaluator = GSgnnAccEvaluator(
+    evaluator = GSgnnClassificationEvaluator(
         config.eval_frequency,
         config.eval_metric,
         config.multilabel,

python/graphstorm/config/argument.py

@@ -352,7 +352,7 @@ def verify_arguments(self, is_train):
         _ = self.log_report_frequency
 
         _ = self.task_type
-        # For classification tasks.
+        # For classification/regression tasks.
         if self.task_type in [BUILTIN_TASK_NODE_CLASSIFICATION, BUILTIN_TASK_EDGE_CLASSIFICATION]:
             _ = self.label_field
             _ = self.num_classes
@@ -368,6 +368,7 @@ def verify_arguments(self, is_train):
                               BUILTIN_TASK_LINK_PREDICTION] and is_train:
             _ = self.exclude_training_targets
             _ = self.reverse_edge_types_map
+        # For link prediction tasks.
         if self.task_type == BUILTIN_TASK_LINK_PREDICTION:
             _ = self.gamma
             _ = self.lp_decoder_type

python/graphstorm/eval/__init__.py

@@ -23,8 +23,7 @@
 from .eval_func import SUPPORTED_REGRESSION_METRICS
 from .eval_func import SUPPORTED_LINK_PREDICTION_METRICS
 
-from .evaluator import GSgnnInstanceEvaluator
-from .evaluator import GSgnnLPEvaluator
-from .evaluator import GSgnnMrrLPEvaluator, GSgnnPerEtypeMrrLPEvaluator
-from .evaluator import GSgnnAccEvaluator
+from .evaluator import GSgnnMrrLPEvaluator
+from .evaluator import GSgnnPerEtypeMrrLPEvaluator
+from .evaluator import GSgnnClassificationEvaluator
 from .evaluator import GSgnnRegressionEvaluator

python/graphstorm/eval/eval_func.py

@@ -103,6 +103,12 @@ def __init__(self):
         self.metric_function["mse"] = compute_mse
         self.metric_function["mae"] = compute_mae
 
+        # This is the operator used to measure each metric performance in evaluation
+        self.metric_eval_function = {}
+        self.metric_eval_function["rmse"] = compute_rmse
+        self.metric_eval_function["mse"] = compute_mse
+        self.metric_eval_function["mae"] = compute_mae
+
     def assert_supported_metric(self, metric):
         """ check if the given metric is supported.
         """
@@ -135,6 +141,14 @@ def __init__(self):
         self.metric_comparator = {}
         self.metric_comparator["mrr"] = operator.le
 
+        # This is the operator used to measure each metric performance
+        self.metric_function = {}
+        self.metric_function["mrr"] = compute_mrr
+
+        # This is the operator used to measure each metric performance in evaluation
+        self.metric_eval_function = {}
+        self.metric_eval_function["mrr"] = compute_mrr
+
     def assert_supported_metric(self, metric):
         """ check if the given metric is supported.
         """
@@ -583,3 +597,19 @@ def compute_mae(pred, labels):
 
     diff = th.abs(pred.cpu() - labels.cpu())
     return th.mean(diff).cpu().item()
+
+def compute_mrr(ranking):
+    """ Get link prediction mrr metrics
+
+        Parameters
+        ----------
+        ranking:
+            ranking of each positive edge
+
+        Returns
+        -------
+        link prediction mrr metrics: tensor
+    """
+    logs = th.div(1.0, ranking)
+    metrics = th.tensor(th.div(th.sum(logs),len(logs)))
+    return metrics