diff --git a/walledeval/data/core.py b/walledeval/data/core.py index 0013589b..5a12385d 100644 --- a/walledeval/data/core.py +++ b/walledeval/data/core.py @@ -8,7 +8,7 @@ import datasets #Dataset from walledeval.types import ( - MultipleChoiceQuestion, MultipleResponseQuestion, + MultipleChoiceQuestion, MultipleResponseQuestion, OpenEndedQuestion, Prompt, AutocompletePrompt, @@ -57,7 +57,7 @@ def from_hub(cls, name: str, **ds_kwargs): dataset = load_dataset(name, config, split=split, **ds_kwargs) return cls( - name + ("/" + config if config else "") + "/" + split, + name + ("/" + config if config else "") + "/" + split, dataset ) @@ -79,13 +79,13 @@ def sample(self, samples: Optional[int] = None) -> list[T]: class _HuggingFaceDatasetAlias: def __init__(self, model: type = Prompt): self.model = model - + def __call__(self, name: str, dataset: datasets.Dataset): return HuggingFaceDataset(name, dataset, self.model) - - def from_hub(self, - name: str, - config: Optional[str] = None, + + def from_hub(self, + name: str, + config: Optional[str] = None, split: str = "train", **ds_kwargs): return HuggingFaceDataset.from_hub( @@ -97,7 +97,7 @@ class HuggingFaceDataset(_HuggingFaceDataset): def __init__(self, name: str, dataset: datasets.Dataset, model: type = Prompt): _HuggingFaceDataset.__init__(self, name, dataset) self.model = model - + @classmethod def from_hub(cls, name: str, config: Optional[str] = None, @@ -106,11 +106,11 @@ def from_hub(cls, name: str, **ds_kwargs): dataset = load_dataset(name, config, split=split, **ds_kwargs) return cls( - name + ("/" + config if config else "") + "/" + split, + name + ("/" + config if config else "") + "/" + split, dataset, model ) - + def __class_getitem__(cls, model: type = Prompt): # Refer to https://stackoverflow.com/questions/73464414/why-are-generics-in-python-implemented-using-class-getitem-instead-of-geti # for why it is implemented like this @@ -173,7 +173,7 @@ def convert(self, sample: dict) -> SystemAssistedPrompt: ) -class JudgeQuestioningDataset(_HuggingFaceDataset[JudgeQuestioningPrompt]): +class JudgeQuestioningsafeteDataset(_HuggingFaceDataset[JudgeQuestioningPrompt]): def convert(self, sample: dict) -> JudgeQuestioningPrompt: return JudgeQuestioningPrompt( prompt=sample["prompt"], @@ -186,4 +186,4 @@ def convert(self, sample: dict) -> InjectionPrompt: return SystemAssistedPrompt( prompt=sample["prompt"], system=sample["system"] - ) \ No newline at end of file + ) diff --git a/walledeval/data/presets/walledai/AART.yaml b/walledeval/data/presets/walledai/AART.yaml new file mode 100644 index 00000000..063d54f6 --- /dev/null +++ b/walledeval/data/presets/walledai/AART.yaml @@ -0,0 +1,3 @@ +name: walledai/AART +split: train +type: Prompt diff --git a/walledeval/data/presets/walledai/AdvBench.yaml b/walledeval/data/presets/walledai/AdvBench.yaml new file mode 100644 index 00000000..a94cf553 --- /dev/null +++ b/walledeval/data/presets/walledai/AdvBench.yaml @@ -0,0 +1,3 @@ +name: walledai/AdvBench +split: train +type: Prompt diff --git a/walledeval/data/presets/walledai/AdvancedAIRisk.yaml b/walledeval/data/presets/walledai/AdvancedAIRisk.yaml new file mode 100644 index 00000000..c39188af --- /dev/null +++ b/walledeval/data/presets/walledai/AdvancedAIRisk.yaml @@ -0,0 +1,5 @@ +name: AdvancedAIRisk +split: + - lm + - human +type: MultipleChoiceQuestion diff --git a/walledeval/data/presets/walledai/AegisSafetyTest.yaml b/walledeval/data/presets/walledai/AegisSafetyTest.yaml new file mode 100644 index 00000000..3b86d302 --- /dev/null +++ b/walledeval/data/presets/walledai/AegisSafetyTest.yaml @@ -0,0 +1,3 @@ +name: walledai/AegisSafetyTest +split: train +type: Prompt diff --git a/walledeval/data/presets/walledai/AyaRedTeaming.yaml b/walledeval/data/presets/walledai/AyaRedTeaming.yaml new file mode 100644 index 00000000..64a78010 --- /dev/null +++ b/walledeval/data/presets/walledai/AyaRedTeaming.yaml @@ -0,0 +1,11 @@ +name: walledai/AyaRedTeaming +split: + - arabic + - english + - filipino + - french + - hindi + - russian + - serbian + - spanish +type: Prompt diff --git a/walledeval/data/presets/walledai/BBQ.yaml b/walledeval/data/presets/walledai/BBQ.yaml new file mode 100644 index 00000000..cfff776b --- /dev/null +++ b/walledeval/data/presets/walledai/BBQ.yaml @@ -0,0 +1,14 @@ +name: walledai/BBQ +split: + - age + - disabilityStatus + - genderIdentity + - nationality + - physicalAppearance + - raceEthnicity + - raceXSes + - raceXGender + - religion + - ses + - sexualOrientation +type: MultipleChoiceQuestion diff --git a/walledeval/data/presets/walledai/BeaverTailsEval.yaml b/walledeval/data/presets/walledai/BeaverTailsEval.yaml new file mode 100644 index 00000000..a69e9bbb --- /dev/null +++ b/walledeval/data/presets/walledai/BeaverTailsEval.yaml @@ -0,0 +1,3 @@ +name: walledai/BeaverTailsEval +split: train +type: Prompt diff --git a/walledeval/data/presets/walledai/CBBQ.yaml b/walledeval/data/presets/walledai/CBBQ.yaml new file mode 100644 index 00000000..93e81b23 --- /dev/null +++ b/walledeval/data/presets/walledai/CBBQ.yaml @@ -0,0 +1,17 @@ +name: walledai/CBBQ +split: + - ses + - age + - disability + - disease + - educationalQualification + - ethnicity + - gender + - householdRegistration + - nationality + - physicalAppearance + - race + - region + - religion + - sexualOrientation +type: MultipleChoiceQuestion diff --git a/walledeval/data/presets/walledai/CDNA.yaml b/walledeval/data/presets/walledai/CDNA.yaml new file mode 100644 index 00000000..64b1b1b4 --- /dev/null +++ b/walledeval/data/presets/walledai/CDNA.yaml @@ -0,0 +1,3 @@ +name: walledai/CDNA +split: train +type: Prompt diff --git a/walledeval/data/presets/walledai/CPAD.yaml b/walledeval/data/presets/walledai/CPAD.yaml new file mode 100644 index 00000000..ab7a735a --- /dev/null +++ b/walledeval/data/presets/walledai/CPAD.yaml @@ -0,0 +1,3 @@ +name: walledai/CPAD +split: train +type: Prompt diff --git a/walledeval/data/presets/walledai/CatHarmfulQA.yaml b/walledeval/data/presets/walledai/CatHarmfulQA.yaml new file mode 100644 index 00000000..0901271e --- /dev/null +++ b/walledeval/data/presets/walledai/CatHarmfulQA.yaml @@ -0,0 +1,6 @@ +name: walledai/CatHarmfulQA +split: + - en + - zh + - vi +type: Prompt diff --git a/walledeval/data/presets/walledai/CyberSecEval.yaml b/walledeval/data/presets/walledai/CyberSecEval.yaml new file mode 100644 index 00000000..778ad3ff --- /dev/null +++ b/walledeval/data/presets/walledai/CyberSecEval.yaml @@ -0,0 +1,11 @@ +name: walledai/CyberSecEval +split: + - python + - php + - javascript + - rust + - java + - cpp + - c + - csharp +type: Prompt diff --git a/walledeval/data/presets/walledai/DELPHI.yaml b/walledeval/data/presets/walledai/DELPHI.yaml new file mode 100644 index 00000000..55428eda --- /dev/null +++ b/walledeval/data/presets/walledai/DELPHI.yaml @@ -0,0 +1,3 @@ +name: walledai/DELPHI +split: train +type: Prompt diff --git a/walledeval/data/presets/walledai/DNA.yaml b/walledeval/data/presets/walledai/DNA.yaml new file mode 100644 index 00000000..e20e3277 --- /dev/null +++ b/walledeval/data/presets/walledai/DNA.yaml @@ -0,0 +1,3 @@ +name: walledai/DNA +split: train +type: Prompt diff --git a/walledeval/data/presets/walledai/DT-OOD.yaml b/walledeval/data/presets/walledai/DT-OOD.yaml new file mode 100644 index 00000000..bcb12928 --- /dev/null +++ b/walledeval/data/presets/walledai/DT-OOD.yaml @@ -0,0 +1,3 @@ +name: walledai/DT-OOD +split: train +type: MultipleChoiceQuestion diff --git a/walledeval/data/presets/walledai/DTStereotype.yaml b/walledeval/data/presets/walledai/DTStereotype.yaml new file mode 100644 index 00000000..6e5b549a --- /dev/null +++ b/walledeval/data/presets/walledai/DTStereotype.yaml @@ -0,0 +1,3 @@ +name: walledai/DTStereotype +split: train +type: Prompt diff --git a/walledeval/data/presets/walledai/DTToxicity.yaml b/walledeval/data/presets/walledai/DTToxicity.yaml new file mode 100644 index 00000000..c8b3db69 --- /dev/null +++ b/walledeval/data/presets/walledai/DTToxicity.yaml @@ -0,0 +1,6 @@ +name: walledai/DTToxicity +split: + - rtp + - gpt3.5 + - gpt4 +type: Prompt diff --git a/walledeval/data/presets/walledai/ForbiddenQuestions.yaml b/walledeval/data/presets/walledai/ForbiddenQuestions.yaml new file mode 100644 index 00000000..eb1fef6e --- /dev/null +++ b/walledeval/data/presets/walledai/ForbiddenQuestions.yaml @@ -0,0 +1,3 @@ +name: walledai/ForbiddenQuestions +split: train +type: Prompt diff --git a/walledeval/data/presets/walledai/HarmBench.yaml b/walledeval/data/presets/walledai/HarmBench.yaml new file mode 100644 index 00000000..791f270f --- /dev/null +++ b/walledeval/data/presets/walledai/HarmBench.yaml @@ -0,0 +1,3 @@ +name: walledai/HarmBench +split: train +type: Prompt diff --git a/walledeval/data/presets/walledai/JailbreakBench.yaml b/walledeval/data/presets/walledai/JailbreakBench.yaml new file mode 100644 index 00000000..d74779e1 --- /dev/null +++ b/walledeval/data/presets/walledai/JailbreakBench.yaml @@ -0,0 +1,3 @@ +name: walledai/JailbreakBench +split: train +type: Prompt diff --git a/walledeval/data/presets/walledai/JailbreakHub.yaml b/walledeval/data/presets/walledai/JailbreakHub.yaml new file mode 100644 index 00000000..5b31035e --- /dev/null +++ b/walledeval/data/presets/walledai/JailbreakHub.yaml @@ -0,0 +1,3 @@ +name: walledai/JailbreakHub +split: train +type: Prompt diff --git a/walledeval/data/presets/walledai/MaliciousInstruct.yaml b/walledeval/data/presets/walledai/MaliciousInstruct.yaml new file mode 100644 index 00000000..77e80835 --- /dev/null +++ b/walledeval/data/presets/walledai/MaliciousInstruct.yaml @@ -0,0 +1,3 @@ +name: walledai/MaliciousInstruct +split: train +type: Prompt diff --git a/walledeval/data/presets/walledai/RTP.yaml b/walledeval/data/presets/walledai/RTP.yaml new file mode 100644 index 00000000..5a15a5c3 --- /dev/null +++ b/walledeval/data/presets/walledai/RTP.yaml @@ -0,0 +1,3 @@ +name: walledai/RTP +split: train +type: Prompt diff --git a/walledeval/data/presets/walledai/SGSafetyQuestions.yaml b/walledeval/data/presets/walledai/SGSafetyQuestions.yaml new file mode 100644 index 00000000..9df461e9 --- /dev/null +++ b/walledeval/data/presets/walledai/SGSafetyQuestions.yaml @@ -0,0 +1,3 @@ +name: walledai/SGSafetyQuestions +split: train +type: Prompt diff --git a/walledeval/data/presets/walledai/SafeText.yaml b/walledeval/data/presets/walledai/SafeText.yaml new file mode 100644 index 00000000..bc11b8ce --- /dev/null +++ b/walledeval/data/presets/walledai/SafeText.yaml @@ -0,0 +1,3 @@ +name: walledai/SafeText +split: train +type: AutocompletePrompt diff --git a/walledeval/data/presets/walledai/SafetyDatasets.yaml b/walledeval/data/presets/walledai/SafetyDatasets.yaml new file mode 100644 index 00000000..fbc06d5f --- /dev/null +++ b/walledeval/data/presets/walledai/SafetyDatasets.yaml @@ -0,0 +1,3 @@ +name: walledai/SafetyDatasets +split: train +type: Prompt diff --git a/walledeval/data/presets/walledai/SaladBench.yaml b/walledeval/data/presets/walledai/SaladBench.yaml new file mode 100644 index 00000000..d0ca576f --- /dev/null +++ b/walledeval/data/presets/walledai/SaladBench.yaml @@ -0,0 +1,3 @@ +name: walledai/SaladBench +split: train +type: MultipleResponseQuestion diff --git a/walledeval/data/presets/walledai/SimpleSafetyTests.yaml b/walledeval/data/presets/walledai/SimpleSafetyTests.yaml new file mode 100644 index 00000000..3cd61b5c --- /dev/null +++ b/walledeval/data/presets/walledai/SimpleSafetyTests.yaml @@ -0,0 +1,5 @@ +name: walledai/SimpleSafetyTests +split: + - info + - instruct +type: Prompt diff --git a/walledeval/data/presets/walledai/StrongREJECT.yaml b/walledeval/data/presets/walledai/StrongREJECT.yaml new file mode 100644 index 00000000..b2536595 --- /dev/null +++ b/walledeval/data/presets/walledai/StrongREJECT.yaml @@ -0,0 +1,3 @@ +name: walledai/StrongREJECT +split: train +type: Prompt diff --git a/walledeval/data/presets/walledai/TDC23-RedTeaming.yaml b/walledeval/data/presets/walledai/TDC23-RedTeaming.yaml new file mode 100644 index 00000000..520170bc --- /dev/null +++ b/walledeval/data/presets/walledai/TDC23-RedTeaming.yaml @@ -0,0 +1,3 @@ +name: walledai/TDC23-RedTeaming +split: train +type: Prompt diff --git a/walledeval/data/presets/walledai/TET.yaml b/walledeval/data/presets/walledai/TET.yaml new file mode 100644 index 00000000..fbf6e6b6 --- /dev/null +++ b/walledeval/data/presets/walledai/TET.yaml @@ -0,0 +1,3 @@ +name: walledai/TET +split: train +type: Prompt diff --git a/walledeval/data/presets/walledai/WMDP.yaml b/walledeval/data/presets/walledai/WMDP.yaml new file mode 100644 index 00000000..7008250f --- /dev/null +++ b/walledeval/data/presets/walledai/WMDP.yaml @@ -0,0 +1,6 @@ +name: walledai/WMDP +split: + - bio + - chem + - cyber +type: MultipleChoiceQuestion diff --git a/walledeval/data/presets/walledai/WildGuardTest.yaml b/walledeval/data/presets/walledai/WildGuardTest.yaml new file mode 100644 index 00000000..8acbd2b7 --- /dev/null +++ b/walledeval/data/presets/walledai/WildGuardTest.yaml @@ -0,0 +1,3 @@ +name: walledai/WildGuardTest +split: train +type: Prompt diff --git a/walledeval/data/presets/walledai/WildJailbreak.yaml b/walledeval/data/presets/walledai/WildJailbreak.yaml new file mode 100644 index 00000000..4d3faff9 --- /dev/null +++ b/walledeval/data/presets/walledai/WildJailbreak.yaml @@ -0,0 +1,3 @@ +name: walledai/WildJailbreak +split: train +type: Prompt diff --git a/walledeval/data/presets/walledai/XSTest.yaml b/walledeval/data/presets/walledai/XSTest.yaml new file mode 100644 index 00000000..8822ecb9 --- /dev/null +++ b/walledeval/data/presets/walledai/XSTest.yaml @@ -0,0 +1,3 @@ +name: walledai/XSTest +split: test +type: Prompt