try truncating generations

llmfoundry/eval/metrics/nlp.py

@@ -70,6 +70,7 @@ def update(self, batch: Optional[Dict[str, Any]], outputs: List[str],
         if batch is None:
             batch = {}
         for sample_output, sample_labels in zip(outputs, labels):
+            sample_output = sample_output.split('\n')[0]
             prediction_tokens = self.normalize_answer(sample_output).split()
             max_f1 = 0
             for label in sample_labels:

scripts/eval/yamls/hf_eval.yaml

@@ -3,7 +3,7 @@ seed: 1
 precision: fp32
 
 # If you are using one model, put it here:
-model_name_or_path: EleutherAI/gpt-neo-125m
+model_name_or_path: mosaicml/mpt-7b-instruct
 # otherwise, write a block for each model you want to test in the `models` section
 
 models:
@@ -12,7 +12,7 @@ models:
   model:
     name: hf_causal_lm
     pretrained_model_name_or_path: ${model_name_or_path}
-    init_device: mixed
+    init_device: cpu
     pretrained: true
   tokenizer:
     name: ${model_name_or_path}
@@ -43,5 +43,32 @@ device_eval_batch_size: 4
 #   forward_prefetch: True
 #   limit_all_gathers: True
 
-icl_tasks: 'eval/yamls/tasks_v0.1.yaml'
-eval_gauntlet: 'eval/yamls/eval_gauntlet_v0.1.yaml'
+icl_tasks:
+-
+    label: quac
+    dataset_uri: eval/local_data/free_response/quac.jsonl
+    num_fewshot: [0]
+    icl_task_type: question_answering
+    metric_names:
+    - 'InContextLearningGenerationF1Score'
+-
+  label: natural_questions_closed
+  dataset_uri: eval/local_data/free_response/natural_questions_closed.jsonl
+  num_fewshot: [0]
+  icl_task_type: question_answering
+  metric_names:
+  - 'InContextLearningGenerationF1Score'
+-
+  label: natural_questions_openbook_short
+  dataset_uri: eval/local_data/free_response/natural_questions_openbook_short.jsonl
+  num_fewshot: [0]
+  icl_task_type: question_answering
+  metric_names:
+  - 'InContextLearningGenerationF1Score'
+-
+  label: narrativeqa
+  dataset_uri: eval/local_data/free_response/narrativeqa.jsonl
+  num_fewshot: [0]
+  icl_task_type: question_answering
+  metric_names:
+  - 'InContextLearningGenerationF1Score'

tests/a_scripts/eval/test_metrics.py

@@ -0,0 +1,17 @@
+from llmfoundry.eval.metrics.nlp import InContextLearningGenerationF1Score
+
+
+
+def test_f1():
+    samples = [
+        {"context": "Title: Anna Politkovskaya\n\nBackground: Anna Stepanovna Politkovskaya (Russian: Anna Stepanovna Politkovskaia, IPA: ['an:@ sjtjI'pan@vn@ p@ljIt'kofsk@j@]; Ukrainian: Ganna Stepanivna Politkovs'ka ['han:a ste'panjiuna poljit'kousjka]; nee Mazepa [ma'zepa]; 30 August 1958 - 7 October 2006) was a Russian journalist, writer, and human rights activist who reported on political events in Russia, in particular, the Second Chechen War (1999-2005).\n\nSection: The murder remains unsolved, 2016\nContext: In September 2016 Vladimir Markin, official spokesman for the Investigative Committee, included the killing of Anna Politkovskaya among the Most Dramatic Crimes in 21st century Russia and claimed that it had been solved. Her colleagues at Novaya gazeta protested that until the instigator or sponsor of the crime was identified, arrested and prosecuted the case was not closed. On 7 October 2016 Novaya gazeta released a video clip of its editors, correspondents, photographers and technical and administrative staff holding text-boards giving details of the case and stating, repeatedly, \"The sponsor of Anna's murder has not been found\". On the same day deputy chief editor Sergei Sokolov published a damning summary of the official investigation, describing its false turns and shortcomings, and emphasised that it had now effectively been wound up. After the three Makhmudov brothers, Khadjikurbanov and Lom-Ali Gaitukayev were convicted in 2014, wrote Sokolov, the once large team of investigators was reduced to one person and within a year he retired, to be replaced by a lower-ranking investigator. In accordance with Russian law there is a 15-year statute of limitation for the \"particularly grave\" crime of first degree murder. The 2000 killing of Igor Domnikov, another Novaya gazeta journalist, showed that the perpetrators might be identified (they were convicted in 2008), as was the businessman-intermediary who hired them (he was sentenced in December 2013 to seven years' imprisonment). The man allegedly responsible for ordering the attack on Domnikov was brought to court in 2015. In May that year the case against him was discontinued because the statute of limitations had expired. The Intercept published a top-secret document released by Edward Snowden with a screenshot of Intellipedia according to which (TS//SI/REL TO USA, AUS, CAN, GBR, NZL) Russian Federal Intelligence Services (probably FSB) are known to have targeted the webmail account of the murdered Russian journalist Anna Politkovskaya. On 5 December 2005, RFIS initiated an attack against the account annapolitovskaya@US Provider1, by deploying malicious software which is not available in the public domain. It is not known whether the attack is in any way associated with the death of the journalist.[1] CANNOTANSWER\n\nQuestion: Did they have any clues?\nAnswer: probably FSB) are known to have targeted the webmail account of the murdered Russian journalist Anna Politkovskaya.\n\nQuestion: How did they target her email?\nAnswer: On 5 December 2005, RFIS initiated an attack against the account annapolitovskaya@US Provider1, by deploying malicious software\n\nQuestion: Did they have any murder suspects?", "answer": "After the three Makhmudov brothers, Khadjikurbanov and Lom-Ali Gaitukayev were convicted in 2014,", "aliases": []},
+        {"context": "Title: Superstar Billy Graham\n\nBackground: Wayne Coleman was born into a working-class family in Phoenix, Arizona on June 07, 1943. His father was from Mississippi, and his mother, of Cherokee ancestry, was from Arkansas. Coleman was attracted to weight lifting in the fifth grade. As a teenager he was an avid reader of bodybuilding magazines, his idols being Steve Reeves and John Grimek.\n\nSection: Disputes with the McMahons\nContext: In the early 1990s US federal agents were investigating Dr. George Zahorian, a Harrisburg, Pennsylvania physician who had been dispensing anabolic steroids and other drugs to World Wrestling Federation (WWF) wrestlers at WWF events. In 1991 Dr. Zahorian was convicted under the US federal Anti-Drug Abuse Act of 1988 which prohibited the prescription of steroids for non-therapeutic purposes. This led to WWF owner Vince McMahon Jr., who admitted to being a steroid user himself, being put on trial on charges of steroid distribution in 1994. The trial concluded with McMahon's acquittal. During this time Graham personally sued Zahorian and the WWF, claiming that they had forced him to take steroids to maintain his position in the company. His lawsuit was unsuccessful, partly because he had been using steroids for a decade preceding his WWF debut. Recalling the lawsuit on a 2003 episode of WWE Confidential, he attributed the litigation to his bitterness and claimed that he was an innovator of steroid use in the organization. Graham went on a public awareness campaign regarding the dangers of steroids during this time, including an appearance with McMahon on The Phil Donahue Show in 1992. During the Donahue taping Graham claimed to have witnessed WWF officials sexually abuse children. McMahon claimed the abuse had never taken place, and Graham later admitted that he made up the allegations, hoping to extort \"hush money\" out of the WWF. In his autobiography, Graham describes making the allegations as being \"my most shameful moment, not only in the wrestling profession, but in my life\". Graham wrote an apology to McMahon but received no response until his 2002 liver transplant. Five years after being inducted into the WWE Hall of Fame, Graham was released from his consultancy position in World Wrestling Entertainment. He sold his Hall of Fame ring on eBay to help pay for medical bills and requested that he be entirely removed from the Hall of Fame after Abdullah The Butcher was inducted. Graham complained that Abdullah had never wrestled in the promotion. Graham spoke out against Linda McMahon during her 2010 Senate campaign, claiming that she was distancing herself from the racy programming that she profited from while acting as CEO for WWE. Upon learning that his liver condition had worsened, Graham reached out to apologize to the McMahons, even offering to be a spokesman for Linda McMahon's campaign. In July 2015, Graham sent a letter to Vince McMahon requesting to take the position of Dusty Rhodes, who had recently died, at NXT. Graham received criticism from fans due to his timing and opportunism. He defended himself, calling his critics \"evil, sub humans\", and compared a possible reconciliation with McMahon to that of long-time WWE critic Bruno Sammartino, who reconciled with McMahon and was inducted into the WWE Hall of Fame class of 2013. CANNOTANSWER\n\nQuestion: what disputes did he have?\nAnswer: Graham personally sued Zahorian and the WWF,\n\nQuestion: why?\nAnswer: claiming that they had forced him to take steroids to maintain his position in the company.\n\nQuestion: did he win the lawsuit?\nAnswer: His lawsuit was unsuccessful,\n\nQuestion: how did the campaign do?", "answer": "During the Donahue taping Graham claimed to have witnessed WWF officials sexually abuse children.", "aliases": []}
+    ]
+    metric = InContextLearningGenerationF1Score()
+    breakpoint()
+
+    metric.update(
+        batch={},
+        outputs=[s['answer'] for s in samples],
+        labels=[[s['answer']] + s['aliases'] for s in samples]
+    )