WIP add code for debugging

axolotl-ai-cloud · Sep 8, 2023 · e9eae77 · e9eae77
1 parent 6f3216e
commit e9eae77
Show file tree

Hide file tree

Showing 5 changed files with 45 additions and 7 deletions.
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -4,6 +4,21 @@
     // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
     "version": "0.2.0",
     "configurations": [
+        {
+            "name": "Python: Remote Attach",
+            "type": "python",
+            "request": "attach",
+            "connect": {
+                "host": "0.0.0.0",
+                "port": 5678
+            },
+            "pathMappings": [
+                {
+                    "localRoot": "${workspaceFolder}",
+                    "remoteRoot": "/workspace/axolotl/"
+                }
+            ]
+        },
         {
             "name": "train",
             "type": "python",

diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -11,6 +11,8 @@ services:
     # set environment variables
     environment:
       - WANDB_API_KEY=${WANDB_API_KEY}
+    ports:
+      - "5678:5678"
     deploy:
       resources:
         reservations:

diff --git a/examples/llama-2/tiny-random.yml b/examples/llama-2/tiny-random.yml
@@ -19,13 +19,14 @@ strict: false
 datasets:
   # - path: mhenrichsen/alpaca_2k_test
   #   type: alpaca
-  # - path: teknium/GPT4-LLM-Cleaned
-  #   type: alpaca
-  - path: Glavin001/startup-interviews
+  - path: teknium/GPT4-LLM-Cleaned
     type: alpaca
+  # - path: Glavin001/startup-interviews
+  #   type: alpaca
 dataset_prepared_path: last_run_prepared
 # val_set_size: 0.01
-val_set_size: 0.1
+val_set_size: 0.001
+# val_set_size: 0.1
 # output_dir: ./lora-out
 # output_dir: ./lora-2-out
 output_dir: ./lora-5-out
@@ -35,7 +36,7 @@ sequence_len: 2048
 # sequence_len: 256
 # sequence_len: 512
 # sample_packing: true
-sample_packing: false
+sample_packing: false # FIXME: disabled until we can fix the bug in callbacks.py
 
 adapter: lora
 lora_model_dir:
@@ -54,11 +55,13 @@ wandb_log_model:
 gradient_accumulation_steps: 4
 # micro_batch_size: 2
 micro_batch_size: 16
+# micro_batch_size: 24
+# micro_batch_size: 24
 # num_epochs: 3
 # num_epochs: 0.001
 # num_epochs: 0.01
-# num_epochs: 1
-num_epochs: 5
+num_epochs: 1
+# num_epochs: 5
 optimizer: adamw_bnb_8bit
 lr_scheduler: cosine
 learning_rate: 0.0002
@@ -81,6 +84,7 @@ warmup_steps: 10
 eval_steps: 10
 # eval_steps: 20
 # eval_steps: 2
+# eval_steps: 1
 save_steps:
 debug:
 deepspeed:

diff --git a/scripts/finetune.py b/scripts/finetune.py
@@ -28,6 +28,11 @@
 from axolotl.utils.tokenization import check_dataset_labels
 from axolotl.utils.wandb import setup_wandb_env_vars
 
+# import debugpy
+# debugpy.listen(('0.0.0.0', 5678))
+# debugpy.wait_for_client()
+# debugpy.breakpoint()
+
 project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
 src_dir = os.path.join(project_root, "src")
 sys.path.insert(0, src_dir)

diff --git a/src/axolotl/utils/callbacks.py b/src/axolotl/utils/callbacks.py
@@ -6,6 +6,7 @@
 import os
 from typing import TYPE_CHECKING, Dict, List
 
+import itertools
 import evaluate
 import numpy as np
 import pandas as pd
@@ -483,3 +484,14 @@ def log_table_from_dataloader(name: str, table_dataloader):
             return control
 
     return LogPredictionCallback
+
+
+def group_sublists_by(lst: List[int], value: int) -> List[List[int]]:
+    """
+    Group sublists of lst by value
+    """
+    grouped = []
+    for key, group in itertools.groupby(lst, lambda x: x == value):
+        if key:
+            grouped.append(list(group))
+    return grouped