Merge pull request #213 from macrocosm-os/update_model_validation_limit

Aligned model update limits to the number of active competitions

constants/__init__.py

@@ -173,6 +173,27 @@
         epsilon_func=LinearDecay(0.005, 0.0002, 36000),
         max_bytes=29 * 1024 * 1024 * 1024,
     ),
+    # This constraint is not actually used, it is added as a copy
+    # of the 14B-model competition constraint entry.
+    # This is just to keep the size of the constraint dict equal
+    # to the number of competitions so `update_models_limit` is
+    # set correctly below.
+    # This hack will be removed once native support for multi datasets
+    # is implemented in a future release.
+    CompetitionId.B14_MODEL_MULTI_DATASET: ModelConstraints(
+        max_model_parameter_size=13_900_000_000,
+        min_model_parameter_size=13_700_000_000,
+        sequence_length=4096,
+        allowed_architectures=ALLOWED_MODEL_TYPES_2,
+        tokenizer="Xenova/gpt-4",
+        kwargs={
+            "torch_dtype": torch.bfloat16,
+            "attn_implementation": "flash_attention_2",
+        },
+        eval_block_delay=EVAL_BLOCK_DELAY,
+        epsilon_func=LinearDecay(0.005, 0.0002, 36000),
+        max_bytes=29 * 1024 * 1024 * 1024,
+    ),
 }
 
 # Schedule of competitions by block.
@@ -256,7 +277,7 @@
 sample_min = 5
 # Max number of uids that can be either pending eval or currently being evaluated.
 # We allow the sample_min per competition + 10 additional models to be held at any one time.
-updated_models_limit = sample_min * len(MODEL_CONSTRAINTS_BY_COMPETITION_ID) + 10
+updated_models_limit = sample_min * len(MODEL_CONSTRAINTS_BY_COMPETITION_ID_2) + 10
 # time required between updates to the chain.
 chain_update_cadence = dt.timedelta(minutes=20)
 # Number of blocks required between retrying evaluation of a model.