New loaders

.isort.cfg

@@ -1,3 +1,3 @@
 [settings]
-known_third_party = _pytest,dimcat,frictionless,git,importlib_metadata,marshmallow,ms3,pandas,plotly,pytest,setuptools,typing_extensions
+known_third_party = _pytest,dimcat,frictionless,git,importlib_metadata,marshmallow,ms3,music21,pandas,plotly,pytest,setuptools,tqdm,typing_extensions
 profile = black

setup.cfg

@@ -51,7 +51,8 @@ install_requires =
     frictionless[zenodo,pandas]==5.13.1
     importlib-metadata~=6.0.0
     marshmallow==3.19.0
-    ms3>=1.1.1
+    ms3 @ git+https://github.com/johentsch/ms3.git@schema
+    music21==9.1.0
     plotly==5.13.0
     seaborn~=0.12.2
     setuptools~=65.6.3

src/dimcat/base.py

@@ -614,6 +614,7 @@ def deserialize_json_file(json_file) -> DimcatObject:
 class DimcatSettings(DimcatObject):
     """Settings for the dimcat library."""
 
+    default_basepath: str = "~/dimcat_data"
     never_store_unvalidated_data: bool = True
     """setting this to False allows for skipping mandatory validations; set to True for production"""
     recognized_piece_columns: List[str] = dataclass_field(
@@ -622,6 +623,7 @@ class DimcatSettings(DimcatObject):
     """column names that are recognized as piece identifiers and automatically renamed to 'piece' when needed"""
 
     class Schema(DimcatObject.Schema):
+        default_basepath = mm.fields.String(required=True)
         never_store_unvalidated_data = mm.fields.Boolean(required=True)
         recognized_piece_columns = mm.fields.List(mm.fields.String(), required=True)
 
@@ -671,14 +673,14 @@ def make_settings_from_config_file(config_filepath: str) -> DimcatConfig:
     try:
         config = parse_config_file(config_filepath)
     except FileNotFoundError:
-        logger.warning(
+        logger.error(
             f"Config file '{config_filepath}' not found. Falling back to default settings."
         )
         return make_default_settings()
     try:
         return make_settings_from_config_parser(config)
     except Exception as e:
-        logger.warning(
+        logger.error(
             f"Error while parsing config file '{config_filepath}': {e}. Falling back to default settings."
         )
         return make_default_settings()

src/dimcat/data/__init__.py

@@ -30,6 +30,7 @@
     Metadata,
     Notes,
     PieceIndex,
+    ResourceSpecs,
     ResourceStatus,
     Result,
     ResultName,
@@ -51,6 +52,7 @@
     resolve_columns_argument,
     resolve_levels_argument,
     resolve_recognized_piece_columns_argument,
+    resource_specs2resource,
 )
 
 logger = logging.getLogger(__name__)

src/dimcat/data/dataset/base.py

@@ -33,7 +33,6 @@
 
 import frictionless as fl
 import marshmallow as mm
-import ms3
 from dimcat.base import DimcatConfig, DimcatObjectField, FriendlyEnum, get_class
 from dimcat.data.base import Data
 from dimcat.data.resources.base import D, DimcatResource, ResourceStatus, SomeDataframe
@@ -53,12 +52,12 @@
     PackageNotFoundError,
     ResourceNotFoundError,
 )
-from dimcat.utils import check_file_path, check_name, get_default_basepath
+from dimcat.utils import check_file_path, check_name, get_default_basepath, resolve_path
 from typing_extensions import Self
 
 if TYPE_CHECKING:
     from dimcat.data.resources.results import Result
-    from dimcat.steps.base import PipelineStep
+    from dimcat.steps.base import FeatureStep
     from dimcat.steps.pipelines import Pipeline
 
 logger = logging.getLogger(__name__)
@@ -196,7 +195,7 @@ def __init__(
         if package_name is not None:
             self.package_name = package_name
         if basepath is not None:
-            basepath = ms3.resolve_dir(basepath)
+            basepath = resolve_path(basepath)
             self.basepath = basepath
         if descriptor_filepath is not None:
             self.descriptor_filepath = descriptor_filepath
@@ -247,7 +246,7 @@ def basepath(self) -> str:
 
     @basepath.setter
     def basepath(self, basepath: str) -> None:
-        basepath = ms3.resolve_dir(basepath)
+        basepath = resolve_path(basepath)
         if self.status > PackageStatus.NOT_SERIALIZED:
             if basepath == self.basepath:
                 return
@@ -812,6 +811,9 @@ def __getitem__(self, item: str) -> DimcatPackage:
     def __iter__(self) -> Iterator[DimcatPackage]:
         yield from self._packages
 
+    def __len__(self) -> int:
+        return len(self._packages)
+
     def __repr__(self):
         return pformat(self.summary_dict(), sort_dicts=False)
 
@@ -971,7 +973,7 @@ def set_basepath(
         set_packages: bool = True,
     ) -> None:
         """Sets the basepath for all packages in the catalog (if set_packages=True)."""
-        basepath_arg = ms3.resolve_dir(basepath)
+        basepath_arg = resolve_path(basepath)
         if not os.path.isdir(basepath_arg):
             raise ValueError(f"basepath {basepath_arg!r} is not an existing directory.")
         self._basepath = basepath_arg
@@ -1047,6 +1049,8 @@ def from_catalogs(
         new_dataset = cls(**kwargs)
         if pipeline is not None:
             new_dataset._pipeline = pipeline
+        new_dataset.inputs.basepath = inputs.basepath
+        new_dataset.outputs.basepath = outputs.basepath
         new_dataset.inputs.extend(inputs)
         new_dataset.outputs.extend(outputs)
         return new_dataset
@@ -1081,18 +1085,28 @@ def init_object(self, data, **kwargs) -> Dataset:
 
     def __init__(
         self,
+        basepath: Optional[str] = None,
         **kwargs,
     ):
         """The central type of object that all :obj:`PipelineSteps <.PipelineStep>` process and return a copy of.
 
         Args:
             **kwargs: Dataset is cooperative and calls super().__init__(data=dataset, **kwargs)
         """
-        self._inputs = InputsCatalog()
-        self._outputs = OutputsCatalog()
+        if basepath is None:
+            self._inputs = InputsCatalog()
+            self._outputs = OutputsCatalog()
+        else:
+            basepath_arg = resolve_path(basepath)
+            if not os.path.isdir(basepath_arg):
+                raise NotADirectoryError(
+                    f"basepath {basepath_arg!r} is not an existing directory."
+                )
+            self._inputs = InputsCatalog(basepath=basepath_arg)
+            self._outputs = OutputsCatalog(basepath=basepath_arg)
         self._pipeline = None
         self.reset_pipeline()
-        super().__init__(**kwargs)
+        super().__init__(**kwargs)  # calls the Mixin's __init__
 
     def __repr__(self):
         return self.info(return_str=True)
@@ -1160,7 +1174,7 @@ def add_output(
 
     def apply(
         self,
-        step: PipelineStep,
+        step: FeatureStep,
     ) -> Self:
         """Applies a pipeline step to the features it is configured for or, if None, to all active features."""
         return step.process_dataset(self)
@@ -1279,7 +1293,7 @@ def get_metadata(self) -> SomeDataframe:
 
     def load_package(
         self,
-        package: Union[fl.Package, str],
+        package: PackageSpecs,
         package_name: Optional[str] = None,
         **options,
     ):
@@ -1309,7 +1323,7 @@ def load_package(
             f"with basepath {self.inputs.basepath}."
         )
 
-    def load_feature(self, feature: Union[FeatureName, str, DimcatConfig]) -> Feature:
+    def load_feature(self, feature: FeatureSpecs) -> Feature:
         """ToDo: Harmonize with FeatureExtractor"""
         feature = self.get_feature(feature)
         feature.load()

src/dimcat/data/resources/__init__.py

@@ -5,8 +5,10 @@
     DimcatResource,
     IndexField,
     PieceIndex,
+    ResourceSpecs,
     ResourceStatus,
     get_pickle_schema,
+    resource_specs2resource,
 )
 from .features import (
     Annotations,

src/dimcat/data/resources/base.py

@@ -5,6 +5,7 @@
 import zipfile
 from enum import IntEnum, auto
 from functools import cache
+from pathlib import Path
 from pprint import pformat
 from typing import (
     Dict,
@@ -23,7 +24,13 @@
 import pandas as pd
 from dimcat.base import DimcatConfig, get_class, get_setting
 from dimcat.data.base import Data
-from dimcat.utils import check_file_path, check_name, get_default_basepath, replace_ext
+from dimcat.utils import (
+    check_file_path,
+    check_name,
+    get_default_basepath,
+    replace_ext,
+    resolve_path,
+)
 from frictionless import FrictionlessException
 from marshmallow import fields, post_load
 from typing_extensions import Self
@@ -675,10 +682,10 @@ def __init__(
             self.default_groupby = default_groupby
 
         if basepath is not None:
-            basepath = ms3.resolve_dir(basepath)
+            basepath = resolve_path(basepath)
 
         if resource is not None:
-            if isinstance(resource, str):
+            if isinstance(resource, (str, Path)):
                 descriptor_path = check_file_path(
                     resource, extensions=("resource.json", "resource.yaml")
                 )
@@ -760,7 +767,7 @@ def basepath(self) -> str:
 
     @basepath.setter
     def basepath(self, basepath: str):
-        basepath = ms3.resolve_dir(basepath)
+        basepath = resolve_path(basepath)
         if self.is_frozen:
             if basepath == self.basepath:
                 return
@@ -1066,7 +1073,7 @@ def _get_current_status(self) -> ResourceStatus:
         return ResourceStatus.EMPTY
 
     @cache
-    def get_dataframe(self) -> Union[D]:
+    def get_dataframe(self) -> D:
         """
         Load the dataframe from disk based on the descriptor's normpath.
 
@@ -1297,6 +1304,26 @@ def validate(
 
 # endregion DimcatResource
 
+ResourceSpecs: TypeAlias = Union[DimcatResource, str, Path]
+
+
+def resource_specs2resource(resource: ResourceSpecs) -> DimcatResource:
+    """Converts a resource specification to a resource.
+
+    Args:
+        resource: A resource specification.
+
+    Returns:
+        A resource.
+    """
+    if isinstance(resource, DimcatResource):
+        return resource
+    if isinstance(resource, (str, Path)):
+        return DimcatResource(resource)
+    raise TypeError(
+        f"Expected a DimcatResource, str, or Path. Got {type(resource).__name__!r}."
+    )
+
 
 @cache
 def get_pickle_schema(name, init=True):

src/dimcat/data/resources/features.py

@@ -8,7 +8,7 @@
 import marshmallow as mm
 from dimcat.base import DimcatConfig, ObjectEnum, is_subclass_of
 from dimcat.data.resources.base import DimcatResource
-from dimcat.exceptions import FeatureNotProcessableError
+from dimcat.exceptions import ResourceNotProcessableError
 
 logger = logging.getLogger(__name__)
 
@@ -148,5 +148,5 @@ def features_argument2config_list(
         allowed_features = [FeatureName(f) for f in allowed_features]
         for config in configs:
             if config.options_dtype not in allowed_features:
-                raise FeatureNotProcessableError(config.options_dtype)
+                raise ResourceNotProcessableError(config.options_dtype)
     return configs

src/dimcat/data/resources/utils.py

@@ -4,7 +4,7 @@
 import os
 from collections import Counter
 from operator import itemgetter
-from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Set, Tuple, Union
+from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Set, Tuple
 from zipfile import ZipFile
 
 import frictionless as fl
@@ -189,7 +189,7 @@ def infer_schema_from_df(df: SomeDataframe) -> fl.Schema:
 def load_fl_resource(
     fl_resource: fl.Resource,
     index_col: Optional[int | str | List[int | str]] = None,
-    usecols: Optional[Union[int, str, List[int | str]]] = None,
+    usecols: Optional[int | str | List[int | str]] = None,
 ) -> SomeDataframe:
     """Load a dataframe from a :obj:`frictionless.Resource`.
 

src/dimcat/exceptions.py

@@ -41,6 +41,16 @@ class BasePathNotDefinedError(DimcatError):
     }
 
 
+class DuplicateIDError(DimcatError):
+    """optional args: (id, facet)"""
+
+    nargs2message = {
+        0: "An ID was already in use.",
+        1: lambda id: f"The ID {id!r} is already in use.",
+        2: lambda id, facet: f"The ID {id!r} is already in use for facet {facet!r}.",
+    }
+
+
 class EmptyCatalogError(DimcatError):
     nargs2message = {
         0: "The catalog is empty.",
@@ -74,7 +84,30 @@ class EmptyResourceError(DimcatError):
     }
 
 
-class FeatureNotProcessableError(DimcatError):
+class ExcludedFileExtensionError(DimcatError):
+    """optional args: (extension, permissible_extensions)"""
+
+    nargs2message = {
+        0: "A file extension is excluded.",
+        1: lambda extension: f"File extension {extension!r} is excluded.",
+        2: lambda extension, permissible_extensions: f"File extension {extension!r} is excluded. "
+        f"Pass one of {permissible_extensions!r}.",
+    }
+
+
+class NoMuseScoreExecutableSpecifiedError(DimcatError):
+    nargs2message = {
+        0: "No MuseScore executable specified.",
+    }
+
+
+class NoPathsSpecifiedError(DimcatError):
+    nargs2message = {
+        0: "No paths have been specified.",
+    }
+
+
+class ResourceNotProcessableError(DimcatError):
     """optional args: (feature_name,)"""
 
     nargs2message = {

src/dimcat/settings.ini

@@ -1,3 +1,6 @@
+[DEFAULT]
+default_basepath = ~/dimcat_data
+
 [EXCEPTIONS]
 # setting this to False allows for skipping mandatory validations; set to True for production
 never_store_unvalidated_data = False

src/dimcat/steps/__init__.py

@@ -9,9 +9,10 @@
     PitchClassVectors,
     UnitOfAnalysis,
 )
-from .base import PipelineStep
+from .base import FeatureStep
 from .extractors import FeatureExtractor
 from .groupers import CustomPieceGrouper, Grouper
+from .loaders import MuseScoreLoader
 from .pipelines import Pipeline
 
 logger = logging.getLogger(__name__)