filled corpus generated

homepage/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

homepage/README.md

@@ -0,0 +1,32 @@
+# corpus_docs
+Sphinx project template to create a corpus documentation homepage including rendered Jupytext notebooks
+
+## Usage
+
+
+1. `git clone --recurse-submodules [email protected]:DCMLab/corpus_docs.git` will include https://github.com/DCMLab/data_reports 
+   as the submodule `notebooks` 
+1. `pip install -r requirements.txt` will install the requirements for both the Sphinx project and the notebooks.
+1. [set the environment variable] `CORPUS_PATH` to the path where the corpus repo or metarepo is located
+1. [set the environment variable] `ANNOTATED_ONLY` to one of `false|f|0` if non-annotated files are to be included
+1. `make html` builds the homepage which you can view at `_build/html/index.html`.
+
+## Next steps
+
+This sphinx project includes  that need to be completed using the Jinja templating library: https://realpython.com/primer-on-jinja-templating/
+Currently these placeholders include:
+* `repo_name`: repository name as in the URLs, e.g. `beethoven_piano_sonatas`
+* `pretty_repo_name`: title, e.g. `Ludwig van Beethoven - The piano sonatas`
+* `zenodo_badge_id`: ID assigned by Zenodo to ensure that the badge always shows the DOI of the latest version.
+* `example_fname`: one of the pieces' file names (without extension) to be used in the docs (e.g. for file paths)
+* `example_full_title`: full title of the example, to be included in the text, e.g. `the first movement of the first sonata Op. 2 no. 1`
+* `example_subcorpus`: (only for meta-corpora) name of the submodule to which the example belongs
+
+The `index.rst` includes the file `repo_readme.md` which is a dummy file. During the actual building it should be 
+replaced by the actual README of the repo. Nevertheless, the dummy could be used to create this README from the 
+placeholder variables.
+
+`introduction.rst` requires that for metarepos (which include submodules), the `git clone` command be extended with `--recurse-submodules -j8`
+
+`analyses.rst` needs to be adapted such that non-annotated repos (setting yet to be introduced), the `annotations` and `cadences` notebooks are not included.
+Could also be done with two different .rst files for each of the cases.

homepage/_static/custom.css

@@ -0,0 +1,9 @@
+.bd-main .bd-content .bd-article-container {
+  max-width: 100%;  /* default is 60em */
+}
+.bd-page-width {
+    max-width: 100%; /* maximizes overall horizontal space */
+}
+.bd-sidebar-primary {
+    max-width: var(--pst-sidebar-secondary); /* same maximal width for the left side bar as for the right one */
+}

homepage/analyses.rst

@@ -0,0 +1,14 @@
+********
+Analyses
+********
+
+The following notebooks contain a couple of standard analysis and have been rendered for 
+display as a static homepage:
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+   notebooks/overview
+   notebooks/notes_stats
+

homepage/conf.py

@@ -0,0 +1,45 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# For the full list of built-in configuration values, see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Project information -----------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
+
+project = "Claude Debussy – Préludes"
+copyright = '2023, Johannes Hentschel'
+author = 'Johannes Hentschel'
+release = 'v0.9.1'
+
+# -- General configuration ---------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
+
+extensions = [
+    "myst_nb", # rendering Jupyter notebooks
+    "jupyter_sphinx", # rendering interactive Plotly in notebooks
+]
+
+templates_path = ['_templates']
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', '**README.md']
+
+
+# -- Options for HTML output -------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
+
+html_theme = 'pydata_sphinx_theme'
+html_static_path = ['_static']
+
+html_css_files = [
+    'custom.css',
+]
+
+# -- MyST Notebook configuration-----------------------------------------------
+# https://myst-nb.readthedocs.io/en/latest/configuration.html
+
+nb_execution_mode = "cache"
+nb_execution_timeout = 300
+nb_execution_allow_errors = False
+nb_execution_show_tb = True
+# toggle text:
+nb_code_prompt_show = "Show {type}"
+nb_code_prompt_hide = "Hide {type}"

homepage/index.rst

@@ -0,0 +1,13 @@
+.. contents:: README
+   :local:
+
+.. include:: repo_readme.md
+   :parser: myst_parser.sphinx_
+
+.. toctree::
+   :maxdepth: 1
+   :caption: Further information:
+
+   introduction
+   analyses
+

homepage/introduction.rst

@@ -0,0 +1,184 @@
+**********
+How to use
+**********
+
+.. contents:: Contents
+   :local:
+
+Getting the data
+================
+
+With full version history
+-------------------------
+
+The dataset is version-controlled via `git <https://git-scm.com/>`__. In order to download the files with all revisions they have gone through, git needs to be installed on your machine.
+Then you can clone this repository using the command
+
+.. code:: bash
+
+
+   git clone https://github.com/DCMLab/debussy_preludes.git
+
+
+Without full version history
+----------------------------
+
+
+If you are only interested in the current version of the corpus, you can download and unpack:
+
+* `Claude Debussy – Préludes <https://github.com/DCMLab/debussy_preludes/archive/refs/heads/main.zip>`__
+
+
+Data Formats
+============
+
+Each piece in this corpus is represented by four files with identical names, each in its own folder. For example, the first prelude from the first book, *Danseuses de Delphes*, has the following files:
+
+-  ``MS3/l117-01_preludes_danseuses.mscx``: Uncompressed MuseScore file including the music and annotation labels.
+-  ``notes/l117-01_preludes_danseuses.tsv``: A table of all note heads contained in the score and their relevant features (not each of them represents an onset, some are tied together)
+-  ``measures/l117-01_preludes_danseuses.tsv``: A table with relevant information about the measures in the score.
+-  ``harmonies/l117-01_preludes_danseuses.tsv``: A list of the included harmony labels (including cadences and phrases) with their positions in the score.
+
+Opening Scores
+--------------
+
+After navigating to your local copy, you can open the scores in the folder ``MS3`` with the free and open source score editor `MuseScore <https://musescore.org>`__. Please note that the scores have been edited, annotated and tested with `MuseScore 3.6.2 <https://github.com/musescore/MuseScore/releases/tag/v3.6.2>`__. MuseScore 4 has since been released and preliminary tests suggest that it renders them correctly.
+
+Opening TSV files in a spreadsheet
+----------------------------------
+
+Tab-separated value (TSV) files are like Comma-separated value (CSV) files and can be opened with most modern text editors. However, for correctly displaying the columns, you might want to use a spreadsheet or an addon for your favourite text editor. When you use a spreadsheet such as Excel, it might annoy you by interpreting fractions as dates. This can be circumvented by using ``Data --> From Text/CSV`` or the free alternative `LibreOffice Calc <https://www.libreoffice.org/download/download/>`__. Other than that, TSV data can be loaded with every modern programming language.
+
+Loading TSV files in Python
+---------------------------
+
+Since the TSV files contain null values, lists, fractions, and numbers that are to be treated as strings, you may want to use this code to load any TSV files related to this repository (provided you’re doing it in Python). After a quick ``pip install -U ms3`` (requires Python 3.10) you’ll be able to load any TSV like this:
+
+.. code:: python
+
+   import ms3
+
+   labels = ms3.load_tsv('harmonies/l117-01_preludes_danseuses.tsv')
+   notes = ms3.load_tsv('notes/l117-01_preludes_danseuses.tsv')
+
+How to read ``metadata.tsv``
+============================
+
+This section explains the meaning of the columns contained in ``metadata.tsv``.
+
+File information
+----------------
+
++------------------------+------------------------------------------------------------+
+| column                 | content                                                    |
++========================+============================================================+
+| **fname**              | name without extension (for referencing related files)     |
++------------------------+------------------------------------------------------------+
+| **rel_path**           | relative file path of the score, including extension       |
++------------------------+------------------------------------------------------------+
+| **subdirectory**       | folder where the score is located                          |
++------------------------+------------------------------------------------------------+
+| **last_mn**            | last measure number                                        |
++------------------------+------------------------------------------------------------+
+| **last_mn_unfolded**   | number of measures when playing all repeats                |
++------------------------+------------------------------------------------------------+
+| **length_qb**          | length of the piece, measured in quarter notes             |
++------------------------+------------------------------------------------------------+
+| **length_qb_unfolded** | length of the piece when playing all repeats               |
++------------------------+------------------------------------------------------------+
+| **volta_mcs**          | measure counts of first and second endings                 |
++------------------------+------------------------------------------------------------+
+| **all_notes_qb**       | summed up duration of all notes, measured in quarter notes |
++------------------------+------------------------------------------------------------+
+| **n_onsets**           | number of note onsets                                      |
++------------------------+------------------------------------------------------------+
+| **n_onset_positions**  | number of unique note onsets (“slices”)                    |
++------------------------+------------------------------------------------------------+
+
+Composition information
+-----------------------
+
++--------------------+---------------------------+
+| column             | content                   |
++====================+===========================+
+| **composer**       | composer name             |
++--------------------+---------------------------+
+| **workTitle**      | work title                |
++--------------------+---------------------------+
+| **composed_start** | earliest composition date |
++--------------------+---------------------------+
+| **composed_end**   | latest composition date   |
++--------------------+---------------------------+
+| **workNumber**     | Catalogue number(s)       |
++--------------------+---------------------------+
+| **movementNumber** | 1, 2, or 3                |
++--------------------+---------------------------+
+| **movementTitle**  | title of the movement     |
++--------------------+---------------------------+
+
+Score information
+-----------------
+
++-----------------+--------------------------------------------------------+
+| column          | content                                                |
++=================+========================================================+
+| **label_count** | number of chord labels                                 |
++-----------------+--------------------------------------------------------+
+| **KeySig**      | key signature(s) (negative = flats, positive = sharps) |
++-----------------+--------------------------------------------------------+
+| **TimeSig**     | time signature(s)                                      |
++-----------------+--------------------------------------------------------+
+| **musescore**   | MuseScore version                                      |
++-----------------+--------------------------------------------------------+
+| **source**      | URL to the first typesetter’s file                     |
++-----------------+--------------------------------------------------------+
+| **typesetter**  | first typesetter                                       |
++-----------------+--------------------------------------------------------+
+| **annotators**  | creator(s) of the chord labels                         |
++-----------------+--------------------------------------------------------+
+| **reviewers**   | reviewer(s) of the chord labels                        |
++-----------------+--------------------------------------------------------+
+
+Identifiers
+-----------
+
+These columns provide a mapping between multiple identifiers for the sonatas (not for individual movements).
+
++-----------------+------------------------------------------------------------------------------------------------------------+
+| column          | content                                                                                                    |
++=================+============================================================================================================+
+| **wikidata**    | URL of the `WikiData <https://www.wikidata.org/>`__ item                                                   |
++-----------------+------------------------------------------------------------------------------------------------------------+
+| **viaf**        | URL of the Virtual International Authority File (`VIAF <http://viaf.org/>`__) entry                        |
++-----------------+------------------------------------------------------------------------------------------------------------+
+| **musicbrainz** | `MusicBrainz <https://musicbrainz.org/>`__ identifier                                                      |
++-----------------+------------------------------------------------------------------------------------------------------------+
+| **imslp**       | URL to the wiki page within the International Music Score Library Project (`IMSLP <https://imslp.org/>`__) |
++-----------------+------------------------------------------------------------------------------------------------------------+
+
+Generating all TSV files from the scores
+========================================
+
+When you have made changes to the scores and want to update the TSV files accordingly, you can use the following command (provided you have pip-installed `ms3 <https://github.com/johentsch/ms3>`__):
+
+.. code:: python
+
+   ms3 extract -M -N -X -F -D # for measures, notes, expanded harmony labels, form labels, and metadata
+
+If, in addition, you want to generate the reviewed scores with out-of-label notes colored in red, you can do
+
+.. code:: python
+
+   ms3 review -M -N -X -F -D # for extracting measures, notes, expanded harmony labels, form labels, and metadata
+
+By adding the flag ``-c`` to the review command, it will additionally compare the (potentially modified) annotations in the score with the ones currently present in the harmonies TSV files and reflect the comparison in the reviewed scores.
+
+Questions, Suggestions, Corrections, Bug Reports
+================================================
+
+For questions, remarks etc., please `create an issue <https://github.com/DCMLab/debussy_preludes/issues>`__ and feel free to fork and submit pull requests.
+
+License
+=======
+
+Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License (`CC BY-NC-SA 4.0 <https://creativecommons.org/licenses/by-nc-sa/4.0/>`__).

homepage/make.bat

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=.
+set BUILDDIR=build
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.https://www.sphinx-doc.org/
+	exit /b 1
+)
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd