From c9aa91b42be775de6dfad3c00a27f2ff72339239 Mon Sep 17 00:00:00 2001
From: guipenedo <nostrumg@gmail.com>
Date: Mon, 8 Jul 2024 14:46:01 +0200
Subject: [PATCH 01/14] added dummy model

---
 src/lighteval/models/dummy_model.py  | 83 ++++++++++++++++++++++++++++
 src/lighteval/models/model_config.py | 17 +++++-
 src/lighteval/models/model_loader.py | 13 ++++-
 3 files changed, 108 insertions(+), 5 deletions(-)
 create mode 100644 src/lighteval/models/dummy_model.py

diff --git a/src/lighteval/models/dummy_model.py b/src/lighteval/models/dummy_model.py
new file mode 100644
index 000000000..49d53e8c7
--- /dev/null
+++ b/src/lighteval/models/dummy_model.py
@@ -0,0 +1,83 @@
+# MIT License
+import random
+from typing import Optional
+
+# Copyright (c) 2024 The HuggingFace Team
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# inspired by https://github.com/EleutherAI/lm-evaluation-harness/blob/main/lm_eval/models/dummy.py
+
+from lighteval.models.abstract_model import LightevalModel
+from lighteval.models.model_config import EnvConfig
+from lighteval.models.model_output import LoglikelihoodSingleTokenReturn, LoglikelihoodReturn, GenerateReturn
+from lighteval.tasks.requests import LoglikelihoodSingleTokenRequest, LoglikelihoodRollingRequest, LoglikelihoodRequest, \
+    GreedyUntilRequest
+
+
+class DummyModel(LightevalModel):
+    """Dummy model to generate random baselines."""
+
+    def __init__(
+            self,
+            config,
+            env_config: EnvConfig,
+    ):
+        self.config = config
+        self.env_config = env_config
+
+    @property
+    def tokenizer(self):
+        return NotImplemented
+
+    @property
+    def add_special_tokens(self):
+        return NotImplemented
+
+    @property
+    def max_length(self) -> int:
+        return NotImplemented
+
+    def greedy_until(self, requests: list[GreedyUntilRequest], override_bs: Optional[int] = None) -> list[
+        GenerateReturn]:
+        return [GenerateReturn(result="random baseline") for _ in range(len(requests))]
+
+    def loglikelihood(self, requests: list[LoglikelihoodRequest], override_bs: Optional[int] = None) -> list[
+        LoglikelihoodReturn]:
+        return [LoglikelihoodReturn(-random.random()) for _ in requests]
+
+    def loglikelihood_rolling(self, requests: list[LoglikelihoodRollingRequest], override_bs: Optional[int] = None) -> \
+            list[LoglikelihoodReturn]:
+        return [LoglikelihoodReturn(-random.random()) for _ in requests]
+
+    def loglikelihood_single_token(self, requests: list[LoglikelihoodSingleTokenRequest],
+                                   override_bs: Optional[int] = None) -> list[LoglikelihoodSingleTokenReturn]:
+        return [
+            LoglikelihoodSingleTokenReturn(result=[-random.random() for _ in req.tokenized_continuation])
+            for req in requests
+        ]
+
+    def tok_encode(self, str_to_encode: str | list[str], add_special_tokens: Optional[bool] = None):
+        return [1]
+
+    def tok_encode_pair(self, context, continuation):
+        return [1], [2]
+
+    def tok_decode(self, tokens) -> list[str]:
+        return ["random baseline"]
diff --git a/src/lighteval/models/model_config.py b/src/lighteval/models/model_config.py
index b686c9bd7..056c3a984 100644
--- a/src/lighteval/models/model_config.py
+++ b/src/lighteval/models/model_config.py
@@ -22,7 +22,7 @@
 
 from argparse import Namespace
 from dataclasses import dataclass
-from typing import Dict, Optional, Union
+from typing import Dict, Optional, Union, TypeAlias
 
 import torch
 import yaml
@@ -203,6 +203,11 @@ class TGIModelConfig:
     model_id: str
 
 
+@dataclass
+class DummyModelConfig:
+    pass
+
+
 @dataclass
 class InferenceModelConfig:
     model: str
@@ -253,7 +258,10 @@ def nullable_keys() -> list[str]:
         return ["namespace", "env_vars", "image_url"]
 
 
-def create_model_config(args: Namespace, accelerator: Union["Accelerator", None]) -> BaseModelConfig:  # noqa: C901
+ModelConfig: TypeAlias = Union[BaseModelConfig, AdapterModelConfig, DeltaModelConfig, TGIModelConfig, InferenceEndpointModelConfig, DummyModelConfig]
+
+
+def create_model_config(args: Namespace, accelerator: Union["Accelerator", None]) -> ModelConfig:  # noqa: C901
     """
     Create a model configuration based on the provided arguments.
 
@@ -262,7 +270,7 @@ def create_model_config(args: Namespace, accelerator: Union["Accelerator", None]
         accelerator (Union[Accelerator, None]): accelerator to use for model training.
 
     Returns:
-        BaseModelConfig: model configuration.
+        ModelConfig: model configuration.
 
     Raises:
         ValueError: If both an inference server address and model arguments are provided.
@@ -271,6 +279,9 @@ def create_model_config(args: Namespace, accelerator: Union["Accelerator", None]
         ValueError: If a base model is specified when not using delta weights or adapter weights.
     """
     if args.model_args:
+        if args.model_args == "dummy":
+            return DummyModelConfig()
+
         args_dict = {k.split("=")[0]: k.split("=")[1] for k in args.model_args.split(",")}
         args_dict["accelerator"] = accelerator
         args_dict["use_chat_template"] = args.use_chat_template
diff --git a/src/lighteval/models/model_loader.py b/src/lighteval/models/model_loader.py
index e662beac0..e0d9e802c 100644
--- a/src/lighteval/models/model_loader.py
+++ b/src/lighteval/models/model_loader.py
@@ -27,6 +27,7 @@
 from lighteval.models.adapter_model import AdapterModel
 from lighteval.models.base_model import BaseModel
 from lighteval.models.delta_model import DeltaModel
+from lighteval.models.dummy_model import DummyModel
 from lighteval.models.endpoint_model import InferenceEndpointModel
 from lighteval.models.model_config import (
     AdapterModelConfig,
@@ -36,6 +37,7 @@
     InferenceEndpointModelConfig,
     InferenceModelConfig,
     TGIModelConfig,
+    DummyModelConfig
 )
 from lighteval.models.tgi_model import ModelClient
 from lighteval.utils import NO_TGI_ERROR_MSG, is_accelerate_available, is_tgi_available
@@ -54,9 +56,9 @@ class ModelInfo:
 
 
 def load_model(  # noqa: C901
-    config: Union[BaseModelConfig, AdapterModelConfig, DeltaModelConfig, TGIModelConfig, InferenceEndpointModelConfig],
+    config: Union[BaseModelConfig, AdapterModelConfig, DeltaModelConfig, TGIModelConfig, InferenceEndpointModelConfig, DummyModelConfig],
     env_config: EnvConfig,
-) -> Tuple[Union[BaseModel, AdapterModel, DeltaModel, ModelClient], ModelInfo]:
+) -> Tuple[Union[BaseModel, AdapterModel, DeltaModel, ModelClient, DummyModel], ModelInfo]:
     """Will load either a model from an inference server or a model from a checkpoint, depending
     on the config type.
 
@@ -82,6 +84,9 @@ def load_model(  # noqa: C901
     if isinstance(config, BaseModelConfig):
         return load_model_with_accelerate_or_default(config=config, env_config=env_config)
 
+    if isinstance(config, DummyModelConfig):
+        return load_dummy_model(config=config, env_config=env_config)
+
 
 def load_model_with_tgi(config: TGIModelConfig):
     if not is_tgi_available():
@@ -143,3 +148,7 @@ def load_model_with_accelerate_or_default(
     hlog(f"Model info: {model_info}")
 
     return model, model_info
+
+
+def load_dummy_model(config: DummyModelConfig, env_config: EnvConfig):
+    return DummyModel(config=config, env_config=env_config), ModelInfo(model_name="dummy")
\ No newline at end of file

From 40f065b4dc5067f0ede2f4ae3bd66a0d6db597f5 Mon Sep 17 00:00:00 2001
From: guipenedo <nostrumg@gmail.com>
Date: Mon, 8 Jul 2024 15:03:25 +0200
Subject: [PATCH 02/14] fix no tokenizer issues

---
 src/lighteval/models/dummy_model.py | 33 +++++++++++++----------------
 1 file changed, 15 insertions(+), 18 deletions(-)

diff --git a/src/lighteval/models/dummy_model.py b/src/lighteval/models/dummy_model.py
index 49d53e8c7..fe407bd49 100644
--- a/src/lighteval/models/dummy_model.py
+++ b/src/lighteval/models/dummy_model.py
@@ -1,19 +1,17 @@
 # MIT License
-import random
-from typing import Optional
-
+#
 # Copyright (c) 2024 The HuggingFace Team
-
+#
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
-
+#
 # The above copyright notice and this permission notice shall be included in all
 # copies or substantial portions of the Software.
-
+#
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
@@ -24,6 +22,11 @@
 
 # inspired by https://github.com/EleutherAI/lm-evaluation-harness/blob/main/lm_eval/models/dummy.py
 
+import random
+from typing import Optional
+
+from transformers import AutoTokenizer
+
 from lighteval.models.abstract_model import LightevalModel
 from lighteval.models.model_config import EnvConfig
 from lighteval.models.model_output import LoglikelihoodSingleTokenReturn, LoglikelihoodReturn, GenerateReturn
@@ -41,18 +44,21 @@ def __init__(
     ):
         self.config = config
         self.env_config = env_config
+        self._tokenizer = None
 
     @property
     def tokenizer(self):
-        return NotImplemented
+        if not self._tokenizer:
+            self._tokenizer = AutoTokenizer.from_pretrained("gpt2")
+        return self._tokenizer
 
     @property
     def add_special_tokens(self):
-        return NotImplemented
+        return False
 
     @property
     def max_length(self) -> int:
-        return NotImplemented
+        return 2048
 
     def greedy_until(self, requests: list[GreedyUntilRequest], override_bs: Optional[int] = None) -> list[
         GenerateReturn]:
@@ -72,12 +78,3 @@ def loglikelihood_single_token(self, requests: list[LoglikelihoodSingleTokenRequ
             LoglikelihoodSingleTokenReturn(result=[-random.random() for _ in req.tokenized_continuation])
             for req in requests
         ]
-
-    def tok_encode(self, str_to_encode: str | list[str], add_special_tokens: Optional[bool] = None):
-        return [1]
-
-    def tok_encode_pair(self, context, continuation):
-        return [1], [2]
-
-    def tok_decode(self, tokens) -> list[str]:
-        return ["random baseline"]

From 151dcc0ceb2b49e3cf0776b86f7cf8eadee1756c Mon Sep 17 00:00:00 2001
From: guipenedo <nostrumg@gmail.com>
Date: Mon, 8 Jul 2024 15:09:32 +0200
Subject: [PATCH 03/14] fix types issue

---
 src/lighteval/models/dummy_model.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/lighteval/models/dummy_model.py b/src/lighteval/models/dummy_model.py
index fe407bd49..84dc92e41 100644
--- a/src/lighteval/models/dummy_model.py
+++ b/src/lighteval/models/dummy_model.py
@@ -66,11 +66,11 @@ def greedy_until(self, requests: list[GreedyUntilRequest], override_bs: Optional
 
     def loglikelihood(self, requests: list[LoglikelihoodRequest], override_bs: Optional[int] = None) -> list[
         LoglikelihoodReturn]:
-        return [LoglikelihoodReturn(-random.random()) for _ in requests]
+        return [LoglikelihoodReturn((-random.random(), False)) for _ in requests]
 
     def loglikelihood_rolling(self, requests: list[LoglikelihoodRollingRequest], override_bs: Optional[int] = None) -> \
             list[LoglikelihoodReturn]:
-        return [LoglikelihoodReturn(-random.random()) for _ in requests]
+        return [LoglikelihoodReturn((-random.random(), False)) for _ in requests]
 
     def loglikelihood_single_token(self, requests: list[LoglikelihoodSingleTokenRequest],
                                    override_bs: Optional[int] = None) -> list[LoglikelihoodSingleTokenReturn]:

From 94dd43ae9618aca534df685f4dd9a4e3239ddad2 Mon Sep 17 00:00:00 2001
From: guipenedo <nostrumg@gmail.com>
Date: Mon, 8 Jul 2024 15:49:00 +0200
Subject: [PATCH 04/14] fix issue with `truncated_tokens_count`

---
 src/lighteval/models/dummy_model.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/lighteval/models/dummy_model.py b/src/lighteval/models/dummy_model.py
index 84dc92e41..ff7e1e43c 100644
--- a/src/lighteval/models/dummy_model.py
+++ b/src/lighteval/models/dummy_model.py
@@ -66,15 +66,18 @@ def greedy_until(self, requests: list[GreedyUntilRequest], override_bs: Optional
 
     def loglikelihood(self, requests: list[LoglikelihoodRequest], override_bs: Optional[int] = None) -> list[
         LoglikelihoodReturn]:
-        return [LoglikelihoodReturn((-random.random(), False)) for _ in requests]
+        return [LoglikelihoodReturn((-random.random(), False), truncated_tokens_count=0, padded_tokens_count=0)
+                for _ in requests]
 
     def loglikelihood_rolling(self, requests: list[LoglikelihoodRollingRequest], override_bs: Optional[int] = None) -> \
             list[LoglikelihoodReturn]:
-        return [LoglikelihoodReturn((-random.random(), False)) for _ in requests]
+        return [LoglikelihoodReturn((-random.random(), False), truncated_tokens_count=0, padded_tokens_count=0)
+                for _ in requests]
 
     def loglikelihood_single_token(self, requests: list[LoglikelihoodSingleTokenRequest],
                                    override_bs: Optional[int] = None) -> list[LoglikelihoodSingleTokenReturn]:
         return [
-            LoglikelihoodSingleTokenReturn(result=[-random.random() for _ in req.tokenized_continuation])
+            LoglikelihoodSingleTokenReturn(result=[-random.random() for _ in req.tokenized_continuation],
+                                           truncated_tokens_count=0, padded_tokens_count=0)
             for req in requests
         ]

From 2a1e427a56c291a8268bb03ced1eb19e8e8d63cc Mon Sep 17 00:00:00 2001
From: guipenedo <nostrumg@gmail.com>
Date: Mon, 8 Jul 2024 15:54:44 +0200
Subject: [PATCH 05/14] fix issue with `truncated_tokens_count` #2

---
 src/lighteval/models/dummy_model.py  | 7 +++----
 src/lighteval/models/model_output.py | 4 ++--
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/src/lighteval/models/dummy_model.py b/src/lighteval/models/dummy_model.py
index ff7e1e43c..4ea7b5792 100644
--- a/src/lighteval/models/dummy_model.py
+++ b/src/lighteval/models/dummy_model.py
@@ -66,18 +66,17 @@ def greedy_until(self, requests: list[GreedyUntilRequest], override_bs: Optional
 
     def loglikelihood(self, requests: list[LoglikelihoodRequest], override_bs: Optional[int] = None) -> list[
         LoglikelihoodReturn]:
-        return [LoglikelihoodReturn((-random.random(), False), truncated_tokens_count=0, padded_tokens_count=0)
+        return [LoglikelihoodReturn((-random.random(), False))
                 for _ in requests]
 
     def loglikelihood_rolling(self, requests: list[LoglikelihoodRollingRequest], override_bs: Optional[int] = None) -> \
             list[LoglikelihoodReturn]:
-        return [LoglikelihoodReturn((-random.random(), False), truncated_tokens_count=0, padded_tokens_count=0)
+        return [LoglikelihoodReturn((-random.random(), False))
                 for _ in requests]
 
     def loglikelihood_single_token(self, requests: list[LoglikelihoodSingleTokenRequest],
                                    override_bs: Optional[int] = None) -> list[LoglikelihoodSingleTokenReturn]:
         return [
-            LoglikelihoodSingleTokenReturn(result=[-random.random() for _ in req.tokenized_continuation],
-                                           truncated_tokens_count=0, padded_tokens_count=0)
+            LoglikelihoodSingleTokenReturn(result=[-random.random() for _ in req.tokenized_continuation])
             for req in requests
         ]
diff --git a/src/lighteval/models/model_output.py b/src/lighteval/models/model_output.py
index 510278585..ce85c0201 100644
--- a/src/lighteval/models/model_output.py
+++ b/src/lighteval/models/model_output.py
@@ -31,8 +31,8 @@ class ModelReturn:
     result: Union[tuple, list, str]
     input_tokens: list[int] = field(default_factory=list)  # model inputs
     generated_tokens: list[int] = field(default_factory=list)  # model generations
-    truncated_tokens_count: Optional[int] = None  # How many tokens truncated
-    padded_tokens_count: Optional[int] = None  # How many tokens of padding
+    truncated_tokens_count: Optional[int] = 0  # How many tokens truncated
+    padded_tokens_count: Optional[int] = 0  # How many tokens of padding
 
     def get_result_for_eval(self):
         raise NotImplementedError()

From 24d72651deed01acb6bd02ac8a7b03af7d5ee219 Mon Sep 17 00:00:00 2001
From: guipenedo <nostrumg@gmail.com>
Date: Mon, 8 Jul 2024 16:12:19 +0200
Subject: [PATCH 06/14] add seed option

---
 src/lighteval/models/dummy_model.py  | 11 ++++++-----
 src/lighteval/models/model_config.py | 18 ++++++++----------
 2 files changed, 14 insertions(+), 15 deletions(-)

diff --git a/src/lighteval/models/dummy_model.py b/src/lighteval/models/dummy_model.py
index 4ea7b5792..cae1be89a 100644
--- a/src/lighteval/models/dummy_model.py
+++ b/src/lighteval/models/dummy_model.py
@@ -28,7 +28,7 @@
 from transformers import AutoTokenizer
 
 from lighteval.models.abstract_model import LightevalModel
-from lighteval.models.model_config import EnvConfig
+from lighteval.models.model_config import EnvConfig, DummyModelConfig
 from lighteval.models.model_output import LoglikelihoodSingleTokenReturn, LoglikelihoodReturn, GenerateReturn
 from lighteval.tasks.requests import LoglikelihoodSingleTokenRequest, LoglikelihoodRollingRequest, LoglikelihoodRequest, \
     GreedyUntilRequest
@@ -39,11 +39,12 @@ class DummyModel(LightevalModel):
 
     def __init__(
             self,
-            config,
+            config: DummyModelConfig,
             env_config: EnvConfig,
     ):
         self.config = config
         self.env_config = env_config
+        self._random = random.Random(self.config.seed)
         self._tokenizer = None
 
     @property
@@ -66,17 +67,17 @@ def greedy_until(self, requests: list[GreedyUntilRequest], override_bs: Optional
 
     def loglikelihood(self, requests: list[LoglikelihoodRequest], override_bs: Optional[int] = None) -> list[
         LoglikelihoodReturn]:
-        return [LoglikelihoodReturn((-random.random(), False))
+        return [LoglikelihoodReturn((-self._random.random(), False))
                 for _ in requests]
 
     def loglikelihood_rolling(self, requests: list[LoglikelihoodRollingRequest], override_bs: Optional[int] = None) -> \
             list[LoglikelihoodReturn]:
-        return [LoglikelihoodReturn((-random.random(), False))
+        return [LoglikelihoodReturn((-self._random.random(), False))
                 for _ in requests]
 
     def loglikelihood_single_token(self, requests: list[LoglikelihoodSingleTokenRequest],
                                    override_bs: Optional[int] = None) -> list[LoglikelihoodSingleTokenReturn]:
         return [
-            LoglikelihoodSingleTokenReturn(result=[-random.random() for _ in req.tokenized_continuation])
+            LoglikelihoodSingleTokenReturn(result=[-self._random.random() for _ in req.tokenized_continuation])
             for req in requests
         ]
diff --git a/src/lighteval/models/model_config.py b/src/lighteval/models/model_config.py
index 056c3a984..55056d939 100644
--- a/src/lighteval/models/model_config.py
+++ b/src/lighteval/models/model_config.py
@@ -22,7 +22,7 @@
 
 from argparse import Namespace
 from dataclasses import dataclass
-from typing import Dict, Optional, Union, TypeAlias
+from typing import Dict, Optional, Union
 
 import torch
 import yaml
@@ -205,7 +205,7 @@ class TGIModelConfig:
 
 @dataclass
 class DummyModelConfig:
-    pass
+    seed: 42
 
 
 @dataclass
@@ -258,10 +258,7 @@ def nullable_keys() -> list[str]:
         return ["namespace", "env_vars", "image_url"]
 
 
-ModelConfig: TypeAlias = Union[BaseModelConfig, AdapterModelConfig, DeltaModelConfig, TGIModelConfig, InferenceEndpointModelConfig, DummyModelConfig]
-
-
-def create_model_config(args: Namespace, accelerator: Union["Accelerator", None]) -> ModelConfig:  # noqa: C901
+def create_model_config(args: Namespace, accelerator: Union["Accelerator", None]) -> Union[BaseModelConfig, AdapterModelConfig, DeltaModelConfig, TGIModelConfig, InferenceEndpointModelConfig, DummyModelConfig]:  # noqa: C901
     """
     Create a model configuration based on the provided arguments.
 
@@ -270,7 +267,7 @@ def create_model_config(args: Namespace, accelerator: Union["Accelerator", None]
         accelerator (Union[Accelerator, None]): accelerator to use for model training.
 
     Returns:
-        ModelConfig: model configuration.
+        Union[BaseModelConfig, AdapterModelConfig, DeltaModelConfig, TGIModelConfig, InferenceEndpointModelConfig, DummyModelConfig]: model configuration.
 
     Raises:
         ValueError: If both an inference server address and model arguments are provided.
@@ -279,10 +276,11 @@ def create_model_config(args: Namespace, accelerator: Union["Accelerator", None]
         ValueError: If a base model is specified when not using delta weights or adapter weights.
     """
     if args.model_args:
-        if args.model_args == "dummy":
-            return DummyModelConfig()
+        args_dict = {k.split("=")[0]: k.split("=")[1] if "=" in k else True for k in args.model_args.split(",")}
+
+        if args_dict.pop("dummy", False):
+            return DummyModelConfig(**args_dict)
 
-        args_dict = {k.split("=")[0]: k.split("=")[1] for k in args.model_args.split(",")}
         args_dict["accelerator"] = accelerator
         args_dict["use_chat_template"] = args.use_chat_template
 

From e9553cebf918bbcc87f3069524839561af19df4b Mon Sep 17 00:00:00 2001
From: guipenedo <nostrumg@gmail.com>
Date: Mon, 8 Jul 2024 16:23:23 +0200
Subject: [PATCH 07/14] fix seed declaration

---
 src/lighteval/models/model_config.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lighteval/models/model_config.py b/src/lighteval/models/model_config.py
index 55056d939..1d5fc323e 100644
--- a/src/lighteval/models/model_config.py
+++ b/src/lighteval/models/model_config.py
@@ -205,7 +205,7 @@ class TGIModelConfig:
 
 @dataclass
 class DummyModelConfig:
-    seed: 42
+    seed: int = 42
 
 
 @dataclass

From 311d68b5e7ae0e73bfd1b6d2f01f18e624b5728f Mon Sep 17 00:00:00 2001
From: guipenedo <nostrumg@gmail.com>
Date: Mon, 8 Jul 2024 16:25:22 +0200
Subject: [PATCH 08/14] add seed to modal_sha

---
 src/lighteval/models/model_loader.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lighteval/models/model_loader.py b/src/lighteval/models/model_loader.py
index e0d9e802c..e67e1d7dd 100644
--- a/src/lighteval/models/model_loader.py
+++ b/src/lighteval/models/model_loader.py
@@ -151,4 +151,4 @@ def load_model_with_accelerate_or_default(
 
 
 def load_dummy_model(config: DummyModelConfig, env_config: EnvConfig):
-    return DummyModel(config=config, env_config=env_config), ModelInfo(model_name="dummy")
\ No newline at end of file
+    return DummyModel(config=config, env_config=env_config), ModelInfo(model_name="dummy", model_sha=str(config.seed))
\ No newline at end of file

From baafc493ca2dfa279966d4795cfab402bbea8964 Mon Sep 17 00:00:00 2001
From: guipenedo <nostrumg@gmail.com>
Date: Mon, 8 Jul 2024 16:26:46 +0200
Subject: [PATCH 09/14] run ruff

---
 src/lighteval/models/dummy_model.py  | 12 ++++++++----
 src/lighteval/models/model_loader.py |  4 ++--
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/src/lighteval/models/dummy_model.py b/src/lighteval/models/dummy_model.py
index cae1be89a..c6c26f601 100644
--- a/src/lighteval/models/dummy_model.py
+++ b/src/lighteval/models/dummy_model.py
@@ -28,10 +28,14 @@
 from transformers import AutoTokenizer
 
 from lighteval.models.abstract_model import LightevalModel
-from lighteval.models.model_config import EnvConfig, DummyModelConfig
-from lighteval.models.model_output import LoglikelihoodSingleTokenReturn, LoglikelihoodReturn, GenerateReturn
-from lighteval.tasks.requests import LoglikelihoodSingleTokenRequest, LoglikelihoodRollingRequest, LoglikelihoodRequest, \
-    GreedyUntilRequest
+from lighteval.models.model_config import DummyModelConfig, EnvConfig
+from lighteval.models.model_output import GenerateReturn, LoglikelihoodReturn, LoglikelihoodSingleTokenReturn
+from lighteval.tasks.requests import (
+    GreedyUntilRequest,
+    LoglikelihoodRequest,
+    LoglikelihoodRollingRequest,
+    LoglikelihoodSingleTokenRequest,
+)
 
 
 class DummyModel(LightevalModel):
diff --git a/src/lighteval/models/model_loader.py b/src/lighteval/models/model_loader.py
index e67e1d7dd..531616448 100644
--- a/src/lighteval/models/model_loader.py
+++ b/src/lighteval/models/model_loader.py
@@ -33,11 +33,11 @@
     AdapterModelConfig,
     BaseModelConfig,
     DeltaModelConfig,
+    DummyModelConfig,
     EnvConfig,
     InferenceEndpointModelConfig,
     InferenceModelConfig,
     TGIModelConfig,
-    DummyModelConfig
 )
 from lighteval.models.tgi_model import ModelClient
 from lighteval.utils import NO_TGI_ERROR_MSG, is_accelerate_available, is_tgi_available
@@ -151,4 +151,4 @@ def load_model_with_accelerate_or_default(
 
 
 def load_dummy_model(config: DummyModelConfig, env_config: EnvConfig):
-    return DummyModel(config=config, env_config=env_config), ModelInfo(model_name="dummy", model_sha=str(config.seed))
\ No newline at end of file
+    return DummyModel(config=config, env_config=env_config), ModelInfo(model_name="dummy", model_sha=str(config.seed))

From 6feef85e9239daf694082ceef0944557d20c758a Mon Sep 17 00:00:00 2001
From: guipenedo <nostrumg@gmail.com>
Date: Mon, 8 Jul 2024 16:30:48 +0200
Subject: [PATCH 10/14] ruff again

---
 src/lighteval/models/dummy_model.py  | 32 +++++++++++++++-------------
 src/lighteval/models/model_config.py | 11 +++++++++-
 src/lighteval/models/model_loader.py |  9 +++++++-
 3 files changed, 35 insertions(+), 17 deletions(-)

diff --git a/src/lighteval/models/dummy_model.py b/src/lighteval/models/dummy_model.py
index c6c26f601..08335db5f 100644
--- a/src/lighteval/models/dummy_model.py
+++ b/src/lighteval/models/dummy_model.py
@@ -42,9 +42,9 @@ class DummyModel(LightevalModel):
     """Dummy model to generate random baselines."""
 
     def __init__(
-            self,
-            config: DummyModelConfig,
-            env_config: EnvConfig,
+        self,
+        config: DummyModelConfig,
+        env_config: EnvConfig,
     ):
         self.config = config
         self.env_config = env_config
@@ -65,22 +65,24 @@ def add_special_tokens(self):
     def max_length(self) -> int:
         return 2048
 
-    def greedy_until(self, requests: list[GreedyUntilRequest], override_bs: Optional[int] = None) -> list[
-        GenerateReturn]:
+    def greedy_until(
+        self, requests: list[GreedyUntilRequest], override_bs: Optional[int] = None
+    ) -> list[GenerateReturn]:
         return [GenerateReturn(result="random baseline") for _ in range(len(requests))]
 
-    def loglikelihood(self, requests: list[LoglikelihoodRequest], override_bs: Optional[int] = None) -> list[
-        LoglikelihoodReturn]:
-        return [LoglikelihoodReturn((-self._random.random(), False))
-                for _ in requests]
+    def loglikelihood(
+        self, requests: list[LoglikelihoodRequest], override_bs: Optional[int] = None
+    ) -> list[LoglikelihoodReturn]:
+        return [LoglikelihoodReturn((-self._random.random(), False)) for _ in requests]
 
-    def loglikelihood_rolling(self, requests: list[LoglikelihoodRollingRequest], override_bs: Optional[int] = None) -> \
-            list[LoglikelihoodReturn]:
-        return [LoglikelihoodReturn((-self._random.random(), False))
-                for _ in requests]
+    def loglikelihood_rolling(
+        self, requests: list[LoglikelihoodRollingRequest], override_bs: Optional[int] = None
+    ) -> list[LoglikelihoodReturn]:
+        return [LoglikelihoodReturn((-self._random.random(), False)) for _ in requests]
 
-    def loglikelihood_single_token(self, requests: list[LoglikelihoodSingleTokenRequest],
-                                   override_bs: Optional[int] = None) -> list[LoglikelihoodSingleTokenReturn]:
+    def loglikelihood_single_token(
+        self, requests: list[LoglikelihoodSingleTokenRequest], override_bs: Optional[int] = None
+    ) -> list[LoglikelihoodSingleTokenReturn]:
         return [
             LoglikelihoodSingleTokenReturn(result=[-self._random.random() for _ in req.tokenized_continuation])
             for req in requests
diff --git a/src/lighteval/models/model_config.py b/src/lighteval/models/model_config.py
index 1d5fc323e..d9b4d71af 100644
--- a/src/lighteval/models/model_config.py
+++ b/src/lighteval/models/model_config.py
@@ -258,7 +258,16 @@ def nullable_keys() -> list[str]:
         return ["namespace", "env_vars", "image_url"]
 
 
-def create_model_config(args: Namespace, accelerator: Union["Accelerator", None]) -> Union[BaseModelConfig, AdapterModelConfig, DeltaModelConfig, TGIModelConfig, InferenceEndpointModelConfig, DummyModelConfig]:  # noqa: C901
+def create_model_config(
+    args: Namespace, accelerator: Union["Accelerator", None]
+) -> Union[
+    BaseModelConfig,
+    AdapterModelConfig,
+    DeltaModelConfig,
+    TGIModelConfig,
+    InferenceEndpointModelConfig,
+    DummyModelConfig,
+]:  # noqa: C901
     """
     Create a model configuration based on the provided arguments.
 
diff --git a/src/lighteval/models/model_loader.py b/src/lighteval/models/model_loader.py
index 531616448..c72d64038 100644
--- a/src/lighteval/models/model_loader.py
+++ b/src/lighteval/models/model_loader.py
@@ -56,7 +56,14 @@ class ModelInfo:
 
 
 def load_model(  # noqa: C901
-    config: Union[BaseModelConfig, AdapterModelConfig, DeltaModelConfig, TGIModelConfig, InferenceEndpointModelConfig, DummyModelConfig],
+    config: Union[
+        BaseModelConfig,
+        AdapterModelConfig,
+        DeltaModelConfig,
+        TGIModelConfig,
+        InferenceEndpointModelConfig,
+        DummyModelConfig,
+    ],
     env_config: EnvConfig,
 ) -> Tuple[Union[BaseModel, AdapterModel, DeltaModel, ModelClient, DummyModel], ModelInfo]:
     """Will load either a model from an inference server or a model from a checkpoint, depending

From efbf6f0cf2e5aa5091873d1173421aa39bd70e91 Mon Sep 17 00:00:00 2001
From: guipenedo <nostrumg@gmail.com>
Date: Mon, 8 Jul 2024 16:37:25 +0200
Subject: [PATCH 11/14] add noqa

---
 src/lighteval/models/model_loader.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lighteval/models/model_loader.py b/src/lighteval/models/model_loader.py
index c72d64038..8a6c4d585 100644
--- a/src/lighteval/models/model_loader.py
+++ b/src/lighteval/models/model_loader.py
@@ -65,7 +65,7 @@ def load_model(  # noqa: C901
         DummyModelConfig,
     ],
     env_config: EnvConfig,
-) -> Tuple[Union[BaseModel, AdapterModel, DeltaModel, ModelClient, DummyModel], ModelInfo]:
+) -> Tuple[Union[BaseModel, AdapterModel, DeltaModel, ModelClient, DummyModel], ModelInfo]:  # noqa: C901
     """Will load either a model from an inference server or a model from a checkpoint, depending
     on the config type.
 

From ef946c049478c5521788242d40195bc52cfcbbcf Mon Sep 17 00:00:00 2001
From: guipenedo <nostrumg@gmail.com>
Date: Mon, 8 Jul 2024 16:44:00 +0200
Subject: [PATCH 12/14] fix style again...

---
 src/lighteval/models/model_loader.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/lighteval/models/model_loader.py b/src/lighteval/models/model_loader.py
index 8a6c4d585..58c3eccbb 100644
--- a/src/lighteval/models/model_loader.py
+++ b/src/lighteval/models/model_loader.py
@@ -65,7 +65,7 @@ def load_model(  # noqa: C901
         DummyModelConfig,
     ],
     env_config: EnvConfig,
-) -> Tuple[Union[BaseModel, AdapterModel, DeltaModel, ModelClient, DummyModel], ModelInfo]:  # noqa: C901
+) -> Tuple[Union[BaseModel, AdapterModel, DeltaModel, ModelClient, DummyModel], ModelInfo]:
     """Will load either a model from an inference server or a model from a checkpoint, depending
     on the config type.
 
@@ -91,8 +91,7 @@ def load_model(  # noqa: C901
     if isinstance(config, BaseModelConfig):
         return load_model_with_accelerate_or_default(config=config, env_config=env_config)
 
-    if isinstance(config, DummyModelConfig):
-        return load_dummy_model(config=config, env_config=env_config)
+    return load_dummy_model(config=config, env_config=env_config)
 
 
 def load_model_with_tgi(config: TGIModelConfig):

From 3d5aeb3c838f3f5ceb6213a18e6da011e6424052 Mon Sep 17 00:00:00 2001
From: guipenedo <nostrumg@gmail.com>
Date: Mon, 8 Jul 2024 16:57:01 +0200
Subject: [PATCH 13/14] fixed noqa (?)

---
 src/lighteval/models/model_config.py | 4 ++--
 src/lighteval/models/model_loader.py | 3 ++-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/lighteval/models/model_config.py b/src/lighteval/models/model_config.py
index d9b4d71af..b6f4bb5d9 100644
--- a/src/lighteval/models/model_config.py
+++ b/src/lighteval/models/model_config.py
@@ -258,7 +258,7 @@ def nullable_keys() -> list[str]:
         return ["namespace", "env_vars", "image_url"]
 
 
-def create_model_config(
+def create_model_config(  # noqa: C901
     args: Namespace, accelerator: Union["Accelerator", None]
 ) -> Union[
     BaseModelConfig,
@@ -267,7 +267,7 @@ def create_model_config(
     TGIModelConfig,
     InferenceEndpointModelConfig,
     DummyModelConfig,
-]:  # noqa: C901
+]:
     """
     Create a model configuration based on the provided arguments.
 
diff --git a/src/lighteval/models/model_loader.py b/src/lighteval/models/model_loader.py
index 58c3eccbb..c72d64038 100644
--- a/src/lighteval/models/model_loader.py
+++ b/src/lighteval/models/model_loader.py
@@ -91,7 +91,8 @@ def load_model(  # noqa: C901
     if isinstance(config, BaseModelConfig):
         return load_model_with_accelerate_or_default(config=config, env_config=env_config)
 
-    return load_dummy_model(config=config, env_config=env_config)
+    if isinstance(config, DummyModelConfig):
+        return load_dummy_model(config=config, env_config=env_config)
 
 
 def load_model_with_tgi(config: TGIModelConfig):

From b0e8048692a2b30ba53c0bb90d8e971ca73c1be7 Mon Sep 17 00:00:00 2001
From: guipenedo <nostrumg@gmail.com>
Date: Mon, 8 Jul 2024 18:04:05 +0200
Subject: [PATCH 14/14] add docs

---
 README.md | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/README.md b/README.md
index b90fc9764..dc4827735 100644
--- a/README.md
+++ b/README.md
@@ -239,6 +239,17 @@ python run_evals_accelerate.py \
     --output_dir "./evals"
 ```
 
+### Using the dummy model
+To debug or obtain random baseline scores for a given set of tasks, you can use the `dummy` model:
+```shell
+python run_evals_accelerate.py \
+    --model_args "dummy"\
+    --tasks <task parameters> \
+    --output_dir output_dir
+```
+This "model" randomly generates logprobs (for selection/accuracy tasks) and the string "random baseline" for generation tasks.
+You can also select a specific seed for the random logprob values generated by the dummy model: `--model_args "dummy,seed=123"`.
+
 ## Deep thanks
 `lighteval` was originally built on top of the great [Eleuther AI Harness](https://github.com/EleutherAI/lm-evaluation-harness) (we use the latter to power the [Open LLM Leaderboard](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard)). We also took a lot of inspiration from the amazing [HELM](https://crfm.stanford.edu/helm/latest/), notably for metrics.