Skip to content

Commit

Permalink
isort ignore specific lines
Browse files Browse the repository at this point in the history
  • Loading branch information
milocress committed Jan 18, 2024
1 parent 21aa9e3 commit 2148405
Show file tree
Hide file tree
Showing 2 changed files with 10 additions and 8 deletions.
4 changes: 2 additions & 2 deletions llmfoundry/data/finetuning/tasks.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
# Copyright 2022 MosaicML LLM Foundry authors
# SPDX-License-Identifier: Apache-2.0
# isort:skip_file

"""Includes code for task-specific seq-to-seq data formatting.
Expand Down Expand Up @@ -37,7 +36,6 @@ def preprocessing_fn(example: Dict) -> Dict[str, str]:
import os
import warnings
from pathlib import Path
from typing import Any, Callable, Dict, List, Literal, Optional, Union

import datasets as hf_datasets
import huggingface_hub as hf_hub
Expand All @@ -47,6 +45,8 @@ def preprocessing_fn(example: Dict) -> Dict[str, str]:

from llmfoundry.utils.logging_utils import SpecificWarningFilter

from typing import Any, Callable, Dict, List, Literal, Optional, Union # isort:skip

log = logging.getLogger(__name__)

__all__ = ['dataset_constructor']
Expand Down
14 changes: 8 additions & 6 deletions tests/data/test_dataloader.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
# Copyright 2022 MosaicML LLM Foundry authors
# SPDX-License-Identifier: Apache-2.0
# isort: skip_file
import contextlib
import os
import pathlib
Expand All @@ -10,7 +9,6 @@
from argparse import Namespace
from contextlib import nullcontext as does_not_raise
from pathlib import Path
from typing import ContextManager, List, Literal, Optional, Union
from unittest.mock import MagicMock

import pytest
Expand All @@ -24,17 +22,21 @@
from llmfoundry import (build_finetuning_dataloader,
build_text_denoising_dataloader)
from llmfoundry.data import build_dataloader
from llmfoundry.data.finetuning.tasks import (
_ALLOWED_PROMPT_KEYS, _ALLOWED_RESPONSE_KEYS,
DOWNLOADED_FT_DATASETS_DIRPATH, SUPPORTED_EXTENSIONS, ChatFormattedDict,
PromptResponseDict, _tokenize_formatted_example)
from llmfoundry.data.text_data import (ConcatenatedSequenceCollatorWrapper,
build_text_dataloader,
get_tokens_per_batch_func)
from llmfoundry.utils.builders import build_tokenizer
from scripts.data_prep.convert_dataset_hf import main as main_hf
from tests.data_utils import make_tiny_ft_dataset

from llmfoundry.data.finetuning.tasks import ( # isort:skip
_ALLOWED_PROMPT_KEYS, _ALLOWED_RESPONSE_KEYS, # isort:skip
DOWNLOADED_FT_DATASETS_DIRPATH, SUPPORTED_EXTENSIONS,
ChatFormattedDict, # isort:skip
PromptResponseDict, _tokenize_formatted_example) # isort:skip

from typing import ContextManager, List, Literal, Optional, Union # isort:skip


def get_config(conf_path: str = 'yamls/mpt/125m.yaml'):
os.environ['TOKENIZERS_PARALLELISM'] = 'false'
Expand Down

0 comments on commit 2148405

Please sign in to comment.