diff --git a/CHANGELOG.md b/CHANGELOG.md index b4bf1b8..b79b669 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ to view arbitrary action segments RGB or Flow contained in a `EpicVideoDataset` using `moviepy`. This is useful when sifting through results per instance. +## Changes +* Improve documentation and bring it all in line with google doc string + standards + # Version 1.5.0 ## Features diff --git a/docs/source/conf.py b/docs/source/conf.py index 2d20eab..6e9d349 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -191,4 +191,10 @@ # -- Options for intersphinx extension --------------------------------------- # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {"https://docs.python.org/3": None} +intersphinx_mapping = { + "python": ("https://docs.python.org/3", None), + "moviepy": ("https://zulko.github.io/moviepy", None), + "numpy": ("http://docs.scipy.org/doc/numpy", None), + "PIL": ("https://pillow.readthedocs.io/en/stable/", None), + "pandas": ("http://pandas.pydata.org/pandas-docs/stable/", None), +} diff --git a/docs/source/index.rst b/docs/source/index.rst index cee14c7..1fdcf9f 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,4 +1,4 @@ -epic-kitchens +epic-kitchens ============= `EPIC Kitchens`_ is an egocentric vision dataset, visit the website to @@ -11,7 +11,7 @@ Have a look at `EPIC-mini`_ for example usage. .. toctree:: - :maxdepth: 2 + :maxdepth: 3 epic_kitchens.rst cli_tools.rst diff --git a/epic_kitchens/__version__.py b/epic_kitchens/__version__.py index 5de612a..037756d 100644 --- a/epic_kitchens/__version__.py +++ b/epic_kitchens/__version__.py @@ -1,6 +1,6 @@ __title__ = "epic-kitchens" __description__ = "EPIC Kitchens action dataset support library" -__version__ = "1.5.0" +__version__ = "1.6.0" __author__ = "EPIC KITCHENS" __author_email__ = "uob-epic-kitchens2018@bristol.ac.uk" __license__ = "MIT" diff --git a/epic_kitchens/dataset/epic_dataset.py b/epic_kitchens/dataset/epic_dataset.py index 3140e55..1036c0c 100644 --- a/epic_kitchens/dataset/epic_dataset.py +++ b/epic_kitchens/dataset/epic_dataset.py @@ -9,6 +9,11 @@ from epic_kitchens.dataset.video_dataset import VideoDataset, VideoSegment +SegmentFilter = Callable[[VideoSegment], bool] +ClassGetter = Callable[[Dict[str, Any]], Any] +VideoTransform = Callable[[List[PIL.Image.Image]], List[PIL.Image.Image]] + + def _verb_class_getter(metadata): return int(metadata[VERB_CLASS_COL]) @@ -38,7 +43,7 @@ def _noun_class_getter(metadata): class GulpVideoSegment(VideoSegment): - """ SegmentRecord for a video segment stored in a gulp file. + """SegmentRecord for a video segment stored in a gulp file. Assumes that the video segment has the following metadata in the gulp file: - id @@ -55,11 +60,12 @@ def __init__( self.gulp_index = gulp_metadata_dict[UID_COL] @property - def id(self): + def id(self) -> str: + """ID of video segment""" return self.gulp_index @property - def label(self): + def label(self) -> Any: cls = self.class_getter(self.metadata) # WARNING: this type check should be removed once we regulp our data # so that classes are ints in the metadata json @@ -70,6 +76,7 @@ def label(self): @property def num_frames(self) -> int: + """Number of video frames""" return self.metadata["num_frames"] def __getitem__(self, item): @@ -90,39 +97,38 @@ def __repr__(self): class EpicVideoDataset(VideoDataset): + """VideoDataset for gulped RGB frames""" + def __init__( self, gulp_path: Union[Path, str], class_type: str, *, with_metadata: bool = False, - class_getter: Optional[Callable[[Dict[str, Any]], Any]] = None, - segment_filter: Optional[Callable[[VideoSegment], bool]] = None, - sample_transform: Optional[ - Callable[[List[PIL.Image.Image]], List[PIL.Image.Image]] - ] = None + class_getter: Optional[ClassGetter] = None, + segment_filter: Optional[SegmentFilter] = None, + sample_transform: Optional[VideoTransform] = None ) -> None: """ + Args: + gulp_path: Path to gulp directory containing the gulped EPIC RGB or flow frames + + class_type: One of verb, noun, verb+noun, None, determines what label the segment + returns. ``None`` should be used for loading test datasets. + + with_metadata: When True the segments will yield a tuple (metadata, class) where the + class is defined by the class getter and the metadata is the raw dictionary stored + in the gulp file. - Parameters - ---------- - gulp_path - Path to gulp directory containing the gulped EPIC RGB or flow frames - class_type - One of verb, noun, verb+noun, None, determines what label the segment returns. - None should be used for loading test datasets - with_metadata - When True the segments will yield a tuple (metadata, class) where the class is defined by the - class getter and the metadata is the raw dictionary stored in the gulp file. - class_getter - Optionally provide a callable that takes in the gulp dict representing the segment from which - you should return the class you wish the segment to have - segment_filter - Optionally provide a callable that takes a segment and returns True if you want to keep the - segment in the dataset, or False if you wish to exclude it - sample_transform - Optionally provide a sample transform function which takes a list of PIL images and transforms - each of them. This is applied on the frames just before returning from load_frames + class_getter: Optionally provide a callable that takes in the gulp dict representing the + segment from which you should return the class you wish the segment to have. + + segment_filter: Optionally provide a callable that takes a segment and returns True if + you want to keep the segment in the dataset, or False if you wish to exclude it. + + sample_transform: Optionally provide a sample transform function which takes a list of + PIL images and transforms each of them. This is applied on the frames just before + returning from :meth:`load_frames`. """ super().__init__( _class_count[class_type], @@ -144,11 +150,26 @@ class getter and the metadata is the raw dictionary stored in the gulp file. @property def video_segments(self) -> List[VideoSegment]: + """ + List of video segments that are present in the dataset. The describe the start and stop + times of the clip and its class. + """ return list(self._video_segments.values()) def load_frames( self, segment: VideoSegment, indices: Optional[Iterable[int]] = None ) -> List[PIL.Image.Image]: + """ + Load frame(s) from gulp directory. + + Args: + segment: Video segment to load + indices: Frames indices to read + + Returns: + Frames indexed by ``indices`` from the ``segment``. + + """ if indices is None: indices = range(0, segment.num_frames) selected_frames = [] # type: List[PIL.Image.Image] @@ -189,6 +210,11 @@ def _sample_video_at_index( class EpicVideoFlowDataset(EpicVideoDataset): + """VideoDataset for loading gulped flow. The loader assumes that flow :math:`u`, :math:`v` + frames are stored alternately in a flat manner: :math:`[u_0, v_0, u_1, v_1, \ldots, u_n, v_n]` + + """ + def _sample_video_at_index( self, record: VideoSegment, index: int ) -> List[PIL.Image.Image]: diff --git a/epic_kitchens/dataset/video_dataset.py b/epic_kitchens/dataset/video_dataset.py index 6d1d547..41ec703 100644 --- a/epic_kitchens/dataset/video_dataset.py +++ b/epic_kitchens/dataset/video_dataset.py @@ -27,7 +27,7 @@ class VideoDataset(ABC): A dataset interface for use with :class:`TsnDataset`. Implement this interface if you wish to use your dataset with TSN. - We cannot use torch.utils.data.Dataset because we need to yield information about + We cannot use :class:`torch.utils.data.Dataset` because we need to yield information about the number of frames per video, which we can't do with the standard torch.utils.data.Dataset. """ diff --git a/epic_kitchens/gulp/__init__.py b/epic_kitchens/gulp/__init__.py index 7a8f3c0..2e7c5bd 100644 --- a/epic_kitchens/gulp/__init__.py +++ b/epic_kitchens/gulp/__init__.py @@ -1,7 +1,7 @@ """ Dataset Adapters for GulpIO. This module contains two adapters for 'gulping' both RGB and flow frames -which can then be used with the ``EpicVideoDataset`` classes. +which can then be used with the :class:`EpicVideoDataset` classes. """ from . import adapter diff --git a/epic_kitchens/gulp/adapter.py b/epic_kitchens/gulp/adapter.py index ed506f7..4fdab6b 100644 --- a/epic_kitchens/gulp/adapter.py +++ b/epic_kitchens/gulp/adapter.py @@ -26,34 +26,38 @@ def __init__( self, video_segment_dir: str, annotations_df: pd.DataFrame, - frame_size=-1, - extension="jpg", - labelled=True, + frame_size: int = -1, + extension: str = "jpg", + labelled: bool = True, ) -> None: """ Gulp all action segments in ``annotations_df`` reading the dumped frames from ``video_segment_dir`` Args: - video_segment_dir: Root directory containing segmented frames:: - - frame-segments/ - ├── P01 - │   ├── P01_01 - │   | ├── P01_01_0_open-door - │   | | ├── frame_0000000008.jpg - │   | | ... - │   | | ├── frame_0000000202.jpg - │   | ... - │   | ├── P01_01_329_put-down-plate - │   | | ├── frame_0000098424.jpg - │   | | ... - │   | | ├── frame_0000098501.jpg - │   ... - - annotations_df: DataFrame containing labels to be gulped. - frame_size (optional): Size of shortest edge of the frame, if not already this size then it will + video_segment_dir: + Root directory containing segmented frames:: + + frame-segments/ + ├── P01 + │   ├── P01_01 + │   | ├── P01_01_0_open-door + │   | | ├── frame_0000000008.jpg + │   | | ... + │   | | ├── frame_0000000202.jpg + │   | ... + │   | ├── P01_01_329_put-down-plate + │   | | ├── frame_0000098424.jpg + │   | | ... + │   | | ├── frame_0000098501.jpg + │   ... + + annotations_df: + DataFrame containing labels to be gulped. + frame_size: + Size of shortest edge of the frame, if not already this size then it will be resized. - extension (optional): Extension of dumped frames. + extension: + Extension of dumped frames. """ self.video_segment_dir = video_segment_dir self.frame_size = int(frame_size) @@ -66,11 +70,12 @@ def iter_data(self, slice_element=None) -> Iterator[Result]: Args: slice_element (optional): If not specified all frames for the segment will be returned - Returns: - dictionary with the fields: + Yields: + dict: dictionary with the fields + * ``meta``: All metadata corresponding to the segment, this is the same as the data in the labels csv - * ``frames``: list of :py:class:`PIL.Image` corresponding to the frames specified + * ``frames``: list of :class:`PIL.Image.Image` corresponding to the frames specified in ``slice_element`` * ``id``: UID corresponding to segment """ @@ -128,8 +133,7 @@ def _find_frames(self, folder): class EpicFlowDatasetAdapter(EpicDatasetAdapter): - """Gulp Dataset Adapter for Gulping flow frames extracted from the EPIC-KITCHENS dataset - """ + """Gulp Dataset Adapter for Gulping flow frames extracted from the EPIC-KITCHENS dataset""" def iter_data(self, slice_element=None): slice_element = slice_element or slice(0, len(self)) diff --git a/epic_kitchens/gulp/visualisation.py b/epic_kitchens/gulp/visualisation.py index 2044511..6a2d7fa 100644 --- a/epic_kitchens/gulp/visualisation.py +++ b/epic_kitchens/gulp/visualisation.py @@ -7,9 +7,12 @@ from epic_kitchens.dataset.epic_dataset import EpicVideoDataset -def _grey_to_rgb(frames: np.array) -> np.array: +def _grey_to_rgb(frames: np.ndarray) -> np.ndarray: """ Convert frame(s) from gray (2D array) to RGB (3D array) + + Args: + frames: A single frame or set of frames """ if not (2 <= frames.ndim <= 3): raise ValueError( @@ -23,6 +26,20 @@ def _grey_to_rgb(frames: np.array) -> np.array: def clipify_rgb( frames: List[PIL.Image.Image], *, fps: float = 60. ) -> ImageSequenceClip: + """ + + Args: + frames: A list of frames + fps: FPS of clip + + Returns + ------- + moviepy.editor.ImageSequenceClip + Frames concatenated into clip + + + + """ if frames == 0: raise ValueError("Expected at least one frame, but received none") frames = [np.array(frame) for frame in frames] @@ -34,8 +51,16 @@ def clipify_flow( frames: List[PIL.Image.Image], *, fps: float = 30. ) -> ImageSequenceClip: """ - Destack flow frames, join them side by side and then create an ImageSequenceClip + Destack flow frames, join them side by side and then create a clip for display + + Args: + frames: + A list of alternating :math:`u`, :math:`v` flow frames to join into a video. Even indices should + be :math:`u` flow frames, and odd indices, :math:`v` flow frames. + + fps: float, optional + FPS of generated :py:class:`moviepy.editor.ImageSequenceClip` """ if frames == 0: raise ValueError("Expected at least one frame, but received none") @@ -44,11 +69,12 @@ def clipify_flow( def combine_flow_uv_frames( - uv_frames: Union[List[PIL.Image.Image], np.array], *, method="hstack", width_axis=2 -) -> np.array: - """ - Destack (u, v) frames and concatenate them side by side for display purposes - """ + uv_frames: Union[List[PIL.Image.Image], np.ndarray], + *, + method="hstack", + width_axis=2 +) -> np.ndarray: + """Destack (u, v) frames and concatenate them side by side for display purposes""" if isinstance(uv_frames[0], PIL.Image.Image): uv_frames = list(map(np.array, uv_frames)) u_frames = np.array(uv_frames[::2]) @@ -57,7 +83,7 @@ def combine_flow_uv_frames( return hstack_frames(u_frames, v_frames, width_axis) -def hstack_frames(*frame_sequences: np.array, width_axis: int = 2) -> np.array: +def hstack_frames(*frame_sequences: np.ndarray, width_axis: int = 2) -> np.ndarray: return np.concatenate(frame_sequences, axis=width_axis) @@ -69,14 +95,9 @@ def show(self, uid: Union[int, str], **kwargs) -> ImageSequenceClip: """ Show the given video corresponding to ``uid`` in a HTML5 video element. - Parameters - ---------- - uid: UID of video segment - fps (float): optional, - - Returns - ------- - ImageSequenceClip of the sequence + Args: + uid: UID of video segment + fps (float, optional): FPS of video sequence """ frames = self.dataset.load_frames(self.dataset[uid]) clip = self._clipify_frames(frames, **kwargs) @@ -88,10 +109,14 @@ def _clipify_frames(self, frames: List[PIL.Image.Image], fps: float = 60.): class RgbVisualiser(Visualiser): + """Visualiser for video dataset containing RGB frames""" + def _clipify_frames(self, frames: List[PIL.Image.Image], fps: float = 60.): return clipify_rgb(frames, fps=fps) class FlowVisualiser(Visualiser): + """Visualiser for video dataset containing optical flow :math:`(u, v)` frames""" + def _clipify_frames(self, frames: List[PIL.Image.Image], fps: float = 30): return clipify_flow(frames, fps=fps) diff --git a/epic_kitchens/internal/loading.py b/epic_kitchens/internal/loading.py index 4b6e3e4..ce657f7 100644 --- a/epic_kitchens/internal/loading.py +++ b/epic_kitchens/internal/loading.py @@ -73,6 +73,7 @@ class AnnotationRepository: def __init__( self, version: str = "v1.5.0", local_dir: Optional[Path] = None ) -> None: + self.version = version base_url = "https://github.com/epic-kitchens/annotations/raw/{}/".format( version ) diff --git a/epic_kitchens/_utils.py b/epic_kitchens/internal/utils.py similarity index 100% rename from epic_kitchens/_utils.py rename to epic_kitchens/internal/utils.py diff --git a/epic_kitchens/meta.py b/epic_kitchens/meta.py index f33076f..a47ad92 100644 --- a/epic_kitchens/meta.py +++ b/epic_kitchens/meta.py @@ -9,21 +9,28 @@ _LOG = getLogger(__name__) -_annotation_repository = AnnotationRepository() - +_annotation_repositories = {"v1.5.0": AnnotationRepository()} +_annotation_repository = _annotation_repositories["v1.5.0"] ActionClass = namedtuple("ActionClass", ["verb_class", "noun_class"]) Action = namedtuple("Action", ["verb", "noun"]) -def set_datadir(dir_: Union[str, Path]): - """ - Set download directory +def set_version(version: str): + global _annotation_repository, _annotation_repositories + if version not in _annotation_repositories: + _annotation_repositories[version] = AnnotationRepository( + version=version, local_dir=_annotation_repository.local_dir + ) + _annotation_repository = _annotation_repositories[version] + - Parameters - ---------- - dir_: Path to directory in which to store all downloaded metadata files +def set_datadir(dir_: Union[str, Path]): + """Set download directory + Args: + dir_: + Path to directory in which to store all downloaded metadata files """ _annotation_repository.local_dir = Path(dir_) _LOG.info("Setting data directory to {}".format(dir_)) @@ -31,86 +38,71 @@ def set_datadir(dir_: Union[str, Path]): def get_datadir() -> Path: """ - - Returns - ------- - datadir + Returns: Directory under which any downloaded files are stored, defaults to current working directory - """ return _annotation_repository.local_dir def verb_to_class(verb: str) -> int: """ - Parameters - ---------- - verb: A noun from a narration + Args: + verb: + A noun from a narration - Returns - ------- - class: The corresponding numeric class of the verb if it exists - - Raises - ------ - IndexError: If the verb doesn't belong to any of the verb classes + Returns: + The corresponding numeric class of the verb if it exists + Raises: + IndexError: + If the verb doesn't belong to any of the verb classes """ return _annotation_repository.inverse_verb_lookup()[verb] def noun_to_class(noun: str) -> int: """ + Args: + noun: + A noun from a narration - Parameters - ---------- - noun: A noun from a narration - - Returns - ------- - class: The corresponding numeric class of the noun if it exists - - Raises - ------ - IndexError: If the noun doesn't belong to any of the noun classes + Returns: + The corresponding numeric class of the noun if it exists + Raises: + IndexError: + If the noun doesn't belong to any of the noun classes """ return _annotation_repository.inverse_noun_lookup()[noun] def class_to_verb(cls: int) -> str: """ + Args: + cls: numeric verb class - Parameters - ---------- - cls: numeric verb class - - Returns - ------- - canonical verb representing the class - + Returns: + Canonical verb representing the class - Raises - ------ - IndexError: if ``cls`` is an invalid verb class + Raises: + IndexError: + if ``cls`` is an invalid verb class """ return _annotation_repository.verb_classes()["class_key"].loc[cls] def class_to_noun(cls: int) -> str: """ + Args: + cls: numeric noun class - Parameters - ---------- - cls: numeric noun class + Returns: + Canonical noun representing the class - Returns - ------- - canonical noun representing the class + Raises: + IndexError: + if ``cls`` is an invalid noun class - Raises - ------ - IndexError: if ``cls`` is an invalid noun class """ return _annotation_repository.noun_classes()["class_key"].loc[cls] @@ -120,9 +112,7 @@ def noun_classes() -> pd.DataFrame: Get dataframe containing the mapping between numeric noun classes, the canonical noun of that class and nouns clustered into the class. - Returns - ------- - pd.DataFrame + Returns: Dataframe with the columns: .. include:: meta/noun_classes.rst @@ -135,10 +125,8 @@ def verb_classes() -> pd.DataFrame: Get dataframe containing the mapping between numeric verb classes, the canonical verb of that class and verbs clustered into the class. - Returns - ------- - pd.DataFrame - Dataframe with the columns: + Returns: + Dataframe with the columns .. include:: meta/verb_classes.rst """ @@ -147,35 +135,23 @@ def verb_classes() -> pd.DataFrame: def many_shot_verbs() -> Set[int]: """ - - Returns - ------- - many_shot_verbs : Set[int] + Returns: The set of verb classes that are many shot (appear more than 100 times in training). - """ return set(_annotation_repository.many_shot_verbs()) def many_shot_nouns() -> Set[int]: """ - - Returns - ------- - many_shot_nouns : Set[int] + Returns: The set of noun classes that are many shot (appear more than 100 times in training). - """ return set(_annotation_repository.many_shot_nouns()) def many_shot_actions() -> Set[ActionClass]: """ - - Returns - ------- - many_shot_actions : Set[ActionClass] - + Returns: The set of actions classes that are many shot (verb_class appears more than 100 times in training, noun_class appears more than 100 times in training, and the action appears at least once in training). @@ -186,46 +162,33 @@ def many_shot_actions() -> Set[ActionClass]: def is_many_shot_action(action_class: ActionClass) -> bool: """ + Args: + action_class: + ``(verb_class, noun_class)`` tuple - Parameters - ---------- - action_class - - Returns - ------- - bool + Returns: Whether action_class is many shot or not - """ return action_class in _annotation_repository.many_shot_actions() def is_many_shot_verb(verb_class: int) -> bool: """ + Args: + verb_class: numeric verb class - Parameters - ---------- - verb_class - - Returns - ------- - bool + Returns: Whether verb_class is many shot or not - """ return verb_class in _annotation_repository.many_shot_verbs() def is_many_shot_noun(noun_class: int) -> bool: """ + Args: + noun_class: numeric noun class - Parameters - ---------- - noun_class - - Returns - ------- - bool + Returns: Whether noun class is many shot or not """ @@ -234,60 +197,43 @@ def is_many_shot_noun(noun_class: int) -> bool: def training_narrations() -> pd.DataFrame: """ - - Returns - ------- - pd.DataFrame - Dataframe with the columns: + Returns: + Dataframe with the columns .. include:: meta/train_action_narrations.rst - """ return _annotation_repository.train_action_narrations() def training_labels() -> pd.DataFrame: """ - - Returns - ------- - training_labels : pd.DataFrame - Dataframe with the columns: + Returns: + Dataframe with the columns .. include:: meta/train_action_labels.rst - - """ return _annotation_repository.train_action_labels().copy() def training_object_labels() -> pd.DataFrame: """ - Returns - ------- - pd.DataFrame - Dataframe with the columns: + Returns: + Dataframe with the columns .. include:: meta/train_object_labels.rst - """ return _annotation_repository.train_object_labels().copy() def test_timestamps(split: str) -> pd.DataFrame: """ + Args: + split: 'seen', 'unseen', or 'all' (loads both with a 'split' - Parameters - ---------- - split: 'seen', 'unseen', or 'all' (loads both with a 'split' - - Returns - ------- - timestamps : pd.DataFrame - Dataframe with the columns: + Returns: + Dataframe with the columns .. include:: meta/test_timestamps.rst - """ if split == "all": return pd.concat( @@ -306,10 +252,7 @@ def test_timestamps(split: str) -> pd.DataFrame: def video_descriptions() -> pd.DataFrame: """ - - Returns - ------- - video_descriptions : pd.DataFrame + Returns: High level description of the task trying to be accomplished in a video .. include:: meta/video_descriptions.rst @@ -319,13 +262,9 @@ def video_descriptions() -> pd.DataFrame: def video_info() -> pd.DataFrame: """ - - Returns - ------- - video_info : pd.DataFrame + Returns: Technical information stating the resolution, duration and FPS of each video. .. include:: meta/video_info.rst - """ return _annotation_repository.video_info().copy() diff --git a/epic_kitchens/preprocessing/__init__.py b/epic_kitchens/preprocessing/__init__.py index f09f371..d912fbe 100644 --- a/epic_kitchens/preprocessing/__init__.py +++ b/epic_kitchens/preprocessing/__init__.py @@ -1,2 +1 @@ -"""Pre-processing tools to munge data into a format suitable for training -""" +"""Pre-processing tools to munge data into a format suitable for training""" diff --git a/epic_kitchens/preprocessing/split_segments.py b/epic_kitchens/preprocessing/split_segments.py index 5993c6e..f858c31 100644 --- a/epic_kitchens/preprocessing/split_segments.py +++ b/epic_kitchens/preprocessing/split_segments.py @@ -12,7 +12,12 @@ import pandas as pd from epic_kitchens.labels import VIDEO_ID_COL -from epic_kitchens.video import Modality, FlowModality, RGBModality, split_video_frames +from epic_kitchens.video import ( + ModalityIterator, + FlowModalityIterator, + RGBModalityIterator, + split_video_frames, +) HELP = """\ Process frame dumps, and a set of annotations in a pickled dataframe @@ -115,12 +120,12 @@ def main(args): if args.modality.lower() == "rgb": frame_dirs = [args.frame_dir] links_dirs = [args.links_dir] - modality = RGBModality(fps=fps) # type: Modality + modality = RGBModalityIterator(fps=fps) # type: ModalityIterator elif args.modality.lower() == "flow": axes = ["u", "v"] frame_dirs = [args.frame_dir.joinpath(axis) for axis in axes] links_dirs = [args.links_dir.joinpath(axis) for axis in axes] - modality = FlowModality( + modality = FlowModalityIterator( rgb_fps=fps, stride=int(args.of_stride), dilation=int(args.of_dilation) ) else: diff --git a/epic_kitchens/time.py b/epic_kitchens/time.py index 857b030..865ca01 100644 --- a/epic_kitchens/time.py +++ b/epic_kitchens/time.py @@ -1,4 +1,4 @@ -""" Functions for converting between frames and timestamps """ +"""Functions for converting between frames and timestamps""" import numpy as np _MINUTES_TO_SECONDS = 60 @@ -9,7 +9,7 @@ def timestamp_to_seconds(timestamp: str) -> float: """ Convert a timestamp into total number of seconds Args: - timestamp: formatted as "HH:MM:SS[.FractionalPart]" + timestamp: formatted as ``HH:MM:SS[.FractionalPart]`` Returns: ``timestamp`` converted to seconds @@ -64,12 +64,13 @@ def seconds_to_timestamp(total_seconds: float) -> str: def timestamp_to_frame(timestamp: str, fps: float) -> int: """ Convert timestamp to frame number given the FPS of the extracted frames + Args: - timestamp: formatted as "HH:MM:SS[.FractionalPart]" + timestamp: formatted as ``HH:MM:SS[.FractionalPart]`` fps: frames per second Returns: - frame corresponding to a specific + frame corresponding timestamp Examples: >>> timestamp_to_frame("00:00:00", 29.97) @@ -117,7 +118,6 @@ def flow_frame_count(rgb_frame: int, stride: int, dilation: int) -> int: 2 >>> flow_frame_count(6, 1, 3) 3 - >>> flow_frame_count(7, 1, 1) 6 >>> flow_frame_count(7, 2, 1) diff --git a/epic_kitchens/video.py b/epic_kitchens/video.py index 85b2d91..0fd3d24 100644 --- a/epic_kitchens/video.py +++ b/epic_kitchens/video.py @@ -16,22 +16,24 @@ Timestamp = str -class Modality(ABC): +class ModalityIterator(ABC): """Interface that a modality extracted from video must implement""" def frame_iterator(self, start: Timestamp, stop: Timestamp) -> Iterable[int]: """ Args: - start: start time (timestamp: HH:MM:SS) - stop: stop time (timestamp: HH:MM:SS) + start: start time (timestamp: ``HH:MM:SS[.FractionalPart]``) + stop: stop time (timestamp: ``HH:MM:SS[.FractionalPart]``) Yields: - frame indices corresponding to segment from start time to stop time + Frame indices iterator corresponding to segment from ``start`` to ``stop`` """ raise NotImplementedError -class RGBModality(Modality): +class RGBModalityIterator(ModalityIterator): + """Iterator for RGB frames""" + def __init__(self, fps): self.fps = fps @@ -41,8 +43,17 @@ def frame_iterator(self, start: Timestamp, stop: Timestamp) -> Iterable[int]: return range(start_frame, stop_frame) -class FlowModality(Modality): +class FlowModalityIterator(ModalityIterator): + """Iterator for optical flow :math:`(u, v)` frames""" + def __init__(self, dilation=1, stride=1, bound=20, rgb_fps=59.94): + """ + Args: + dilation: Dilation that optical flow was extracted with + stride: Stride that optical flow was extracted with + bound: Bound that optical flow was extracted with + rgb_fps: FPS of RGB video flow was computed from + """ self.dilation = dilation self.stride = stride self.bound = bound @@ -73,13 +84,25 @@ def iterate_frame_dir(root: Path) -> Iterator[Tuple[Path, Path]]: def split_dataset_frames( - modality: Modality, + modality_iterator: ModalityIterator, frames_dir: Path, segment_root_dir: Path, annotations: pd.DataFrame, frame_format="frame%06d.jpg", pattern=re.compile(".*"), -): +) -> None: + """Split dumped video frames from ``frames_dir`` into directories within ``segment_root_dir`` for each + video segment defined in ``annotations``. + + Args: + modality_iterator: Modality iterator of frames + frames_dir: Directory containing dumped frames + segment_root_dir: Directory to write split segments to + annotations: Dataframe containing segment information + frame_format (str, optional): Old style string format that must contain a single ``%d`` formatter + describing file name format of the dumped frames. + pattern (re.Pattern, optional): Regexp to match video directories + """ assert frames_dir.exists() frames_dir = frames_dir.resolve() @@ -92,7 +115,7 @@ def split_dataset_frames( annotations[VIDEO_ID_COL] == video_dir.name ] split_video_frames( - modality, + modality_iterator, frame_format, annotations_for_video, segment_root_dir, @@ -101,6 +124,7 @@ def split_dataset_frames( def get_narration(annotation): + """Get narration from annotation row, defaults to ``"unnarrated"`` if row has no narration column.""" try: return getattr(annotation, NARRATION_COL) except AttributeError: @@ -108,12 +132,24 @@ def get_narration(annotation): def split_video_frames( - modality: Modality, + modality_iterator: ModalityIterator, frame_format: str, video_annotations: pd.DataFrame, segment_root_dir: Path, video_dir: Path, -): +) -> None: + """Split frames from a single video file stored in ``video_dir`` into segment directories stored + in ``segment_root_dir``. + + Args: + modality_iterator: Modality iterator + frame_format: Old style string format that must contain a single ``%d`` formatter + describing file name format of the dumped frames. + video_annotations: Dataframe containing rows only corresponding to video frames stored in + :param:`video_dir` + segment_root_dir: Directory to write split segments to + video_dir: Directory containing dumped frames for a single video + """ for annotation in video_annotations.itertuples(): segment_dir_name = "{video_id}_{index}_{narration}".format( index=annotation.Index, @@ -124,7 +160,9 @@ def split_video_frames( segment_dir.mkdir(parents=True, exist_ok=True) start_timestamp = getattr(annotation, START_TS_COL) stop_timestamp = getattr(annotation, STOP_TS_COL) - frame_iterator = modality.frame_iterator(start_timestamp, stop_timestamp) + frame_iterator = modality_iterator.frame_iterator( + start_timestamp, stop_timestamp + ) LOG.info( "Linking {video_id} - {narration} - {start}--{stop}".format( diff --git a/tests/unit/test__utils.py b/tests/unit/internal/test_utils.py similarity index 94% rename from tests/unit/test__utils.py rename to tests/unit/internal/test_utils.py index 1adaae2..c2a75ba 100644 --- a/tests/unit/test__utils.py +++ b/tests/unit/internal/test_utils.py @@ -2,7 +2,7 @@ from pathlib import Path from unittest.mock import Mock, MagicMock -from epic_kitchens._utils import before, maybe_download +from epic_kitchens.internal.utils import before, maybe_download URL_ROOT = "https://raw.githubusercontent.com/epic-kitchens/annotations/master/" diff --git a/tests/unit/meta/__init__.py b/tests/unit/meta/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/unit/meta/test_module_functions.py b/tests/unit/meta/test_module_functions.py new file mode 100644 index 0000000..fdfb6c4 --- /dev/null +++ b/tests/unit/meta/test_module_functions.py @@ -0,0 +1,12 @@ +from epic_kitchens import meta + + +def test_default_version_is_1_5_0(): + assert meta._annotation_repository.version == "v1.5.0" + + +def test_changing_annotation_version(): + version = "v1.4.0" + assert meta._annotation_repository.version != version + meta.set_version(version) + assert meta._annotation_repository.version == version