[WIP] Process references

docs/conf.py

@@ -17,29 +17,28 @@
 
 # -- Project information -----------------------------------------------------
 
-project = 'Research References Tracking Tool (R2T2)'
-copyright = '2020, Research Computing Service, Imperial College London'
-author = 'Research Computing Service, Imperial College London'
+project = "Research References Tracking Tool (R2T2)"
+copyright = "2020, Research Computing Service, Imperial College London"
+author = "Research Computing Service, Imperial College London"
 
 # The full version, including alpha/beta/rc tags
-release = '0.3.1'
+release = "0.3.1"
 
 
 # -- General configuration ---------------------------------------------------
 
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = [
-]
+extensions = []
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This pattern also affects html_static_path and html_extra_path.
-exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
 
 # The master toctree document.
 master_doc = "index"
@@ -49,9 +48,9 @@
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'alabaster'
+html_theme = "alabaster"
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+html_static_path = ["_static"]

docs/examples/minimal-class.py

@@ -1,9 +1,11 @@
 from r2t2 import add_reference
 
 
-@add_reference(short_purpose="Original implementation of R2T2",
-               reference="Diego Alonso-Álvarez, et al."
-               "(2018, February 27). Solcore (Version 5.1.0). Zenodo."
-               "http://doi.org/10.5281/zenodo.1185316")
-class MyGreatClass():
+@add_reference(
+    short_purpose="Original implementation of R2T2",
+    reference="Diego Alonso-Álvarez, et al."
+    "(2018, February 27). Solcore (Version 5.1.0). Zenodo."
+    "http://doi.org/10.5281/zenodo.1185316",
+)
+class MyGreatClass:
     pass

docs/examples/minimal-method.py

@@ -1,11 +1,12 @@
 from r2t2 import add_reference
 
 
-class MyGreatClass():
-
-    @add_reference(short_purpose="Original implementation of R2T2",
-                   reference="Diego Alonso-Álvarez, et al."
-                   "(2018, February 27). Solcore (Version 5.1.0). Zenodo."
-                   "http://doi.org/10.5281/zenodo.1185316")
+class MyGreatClass:
+    @add_reference(
+        short_purpose="Original implementation of R2T2",
+        reference="Diego Alonso-Álvarez, et al."
+        "(2018, February 27). Solcore (Version 5.1.0). Zenodo."
+        "http://doi.org/10.5281/zenodo.1185316",
+    )
     def my_great_function(self):
         pass

docs/examples/minimal.py

@@ -1,10 +1,12 @@
 from r2t2 import add_reference
 
 
-@add_reference(short_purpose="Original implementation of R2T2",
-               reference="Diego Alonso-Álvarez, et al."
-               "(2018, February 27). Solcore (Version 5.1.0). Zenodo."
-               "http://doi.org/10.5281/zenodo.1185316")
+@add_reference(
+    short_purpose="Original implementation of R2T2",
+    reference="Diego Alonso-Álvarez, et al."
+    "(2018, February 27). Solcore (Version 5.1.0). Zenodo."
+    "http://doi.org/10.5281/zenodo.1185316",
+)
 def my_great_function():
     pass
 

r2t2/__main__.py

@@ -10,7 +10,7 @@
 from .writers import REGISTERED_WRITERS
 from .docstring_reference_parser import (
     expand_file_list,
-    parse_and_add_docstring_references_from_files
+    parse_and_add_docstring_references_from_files,
 )
 
 
@@ -38,7 +38,7 @@ def add_common_arguments(parser: argparse.ArgumentParser):
         default="terminal",
         type=str,
         choices=sorted(REGISTERED_WRITERS.keys()),
-        help="Format of the output. Default: Terminal."
+        help="Format of the output. Default: Terminal.",
     )
     parser.add_argument(
         "--encoding",
@@ -53,11 +53,7 @@ def add_common_arguments(parser: argparse.ArgumentParser):
         help="File to save the references into. Ignored if format is 'Terminal'."
         " Default: [target folder]/references.",
     )
-    parser.add_argument(
-        "--debug",
-        action="store_true",
-        help="Enable debug logging"
-    )
+    parser.add_argument("--debug", action="store_true", help="Enable debug logging")
 
 
 class RunSubCommand(SubCommand):
@@ -102,27 +98,24 @@ def add_arguments(self, parser: argparse.ArgumentParser):
             "target",
             default=".",
             type=str,
-            help="Target file or folder to analyse."
-            " Default: Current directory.",
+            help="Target file or folder to analyse." " Default: Current directory.",
         )
 
     def run(self, args: argparse.Namespace):
         if args.notebook:
-            if not args.target.endswith('.ipynb'):
-                raise Exception("If --notebook flag is passed, target must be a"
-                                " Jupyter notebook!")
+            if not args.target.endswith(".ipynb"):
+                raise Exception(
+                    "If --notebook flag is passed, target must be a"
+                    " Jupyter notebook!"
+                )
         locate_references(args.target, encoding=args.encoding)
         if args.docstring or args.notebook:
             parse_and_add_docstring_references_from_files(
-                expand_file_list(args.target),
-                encoding=args.encoding
+                expand_file_list(args.target), encoding=args.encoding
             )
 
 
-SUB_COMMANDS: List[SubCommand] = [
-    RunSubCommand(),
-    StaticSubCommand()
-]
+SUB_COMMANDS: List[SubCommand] = [RunSubCommand(), StaticSubCommand()]
 
 SUB_COMMAND_BY_NAME: Dict[str, SubCommand] = {
     sub_command.name: sub_command for sub_command in SUB_COMMANDS
@@ -169,6 +162,6 @@ def main(argv: List[str] = None):
     run(args)
 
 
-if __name__ == '__main__':
-    logging.basicConfig(level='INFO')
+if __name__ == "__main__":
+    logging.basicConfig(level="INFO")
     main()

r2t2/core.py

@@ -3,12 +3,18 @@
 from typing import NamedTuple, List, Optional, Callable, Dict, Union
 from functools import reduce
 from pathlib import Path
+from warnings import warn
+
+import bibtexparser as bp
+
+from .doi2bib import doi2bib
 
 
 class FunctionReference(NamedTuple):
     name: str
     line: int
     source: str
+    package: str
     short_purpose: List[str]
     references: List[str]
 
@@ -19,6 +25,7 @@ class Biblio(dict):
     def __init__(self):
         super().__init__()
         self._sources: Dict[str, Path] = {}
+        self._sources_loaded: Dict[str, bp.bibdatabase.BibDatabase] = {}
 
     def __str__(self):
         def add_record(out, record):
@@ -37,6 +44,7 @@ def add_record(out, record):
     def clear(self) -> None:
         super().clear()
         self._sources.clear()
+        self._sources_loaded.clear()
 
     @property
     def references(self):
@@ -78,6 +86,49 @@ def add_source(self, source: Union[str, Path]) -> None:
             )
         self._sources[package] = src
 
+    def load_source(self, package: str) -> None:
+        """Open the source for the given package."""
+        with self._sources[package].open() as f:
+            self._sources_loaded[package] = bp.load(f)
+
+    def add_entry_to_source(self, entry: dict, package: str) -> None:
+        """Add entry to source and save it source for the given package."""
+        self._sources_loaded[package].entries.append(entry)
+        with self._sources[package].open() as f:
+            bp.dump(self._sources_loaded[package], f)
+
+    def process_ref(self, ref: FunctionReference) -> str:
+        if ref.package not in self._sources_loaded:
+            self.load_source(ref.package)
+
+        for refstr in ref.references:
+            if refstr.startswith("[plain]"):
+                return refstr.strip("[plain]")
+
+            elif refstr.startswith("[bibkey]"):
+                return self._sources_loaded[ref.package].entries_dict[
+                    refstr.strip("[bibkey]")
+                ]
+
+            elif refstr.startswith("[doi]"):
+                for entry in self._sources_loaded[ref.package].entries:
+                    out = entry if entry.get("doi") == refstr.strip("[doi]") else None
+                    if out:
+                        db = bp.bibdatabase.BibDatabase()
+                        db.entries = [out]
+                        return bp.dumps(db)
+
+                out = doi2bib(refstr.strip("[doi]"))
+                if out:
+                    self.add_entry_to_source(bp.loads(out), ref.package)
+                    return out
+
+                warn(
+                    f"Reference with doi={refstr.strip('[doi]')} not found!",
+                    UserWarning,
+                )
+                return ""
+
 
 BIBLIOGRAPHY: Biblio = Biblio()
 
@@ -116,13 +167,17 @@ def wrapper(wrapped, instance, args, kwargs):
         source = inspect.getsourcefile(wrapped)
         line = inspect.getsourcelines(wrapped)[1]
         identifier = f"{source}:{line}"
+        try:
+            package = inspect.getmodule(inspect.stack()[1][0]).__name__.split(".")[0]
+        except AttributeError:
+            package = ""
 
         if identifier in BIBLIOGRAPHY and ref in BIBLIOGRAPHY[identifier].references:
             return wrapped(*args, **kwargs)
 
         if identifier not in BIBLIOGRAPHY:
             BIBLIOGRAPHY[identifier] = FunctionReference(
-                wrapped.__name__, line, source, [], []
+                wrapped.__name__, line, source, package, [], []
             )
 
         BIBLIOGRAPHY[identifier].short_purpose.append(short_purpose)

r2t2/docstring_parser.py

@@ -13,51 +13,53 @@
 """
 
 
-DEFAULT_ENCODING = 'utf-8'
+DEFAULT_ENCODING = "utf-8"
 
 
 class CodeDocumentComment(NamedTuple):
     text: str
     filename: Optional[str] = None
     lineno: Optional[int] = None
     name: Optional[str] = None
+    package: Optional[str] = None
 
 
 def iter_extract_docstring_from_text(
-    text: str, filename: str = None,
+    text: str,
+    filename: str = None,
     notebook: bool = False,
 ) -> Iterable[CodeDocumentComment]:
-    tree = ast.parse(text, filename=filename or '<unknown>')
+    tree = ast.parse(text, filename=filename or "<unknown>")
     for node in ast.walk(tree):
-        LOGGER.debug('node: %r', node)
+        LOGGER.debug("node: %r", node)
         try:
             node_docstring = ast.get_docstring(node)
-            LOGGER.debug('node_docstring: %r', node_docstring)
+            LOGGER.debug("node_docstring: %r", node_docstring)
             if node_docstring:
                 if notebook:
-                    lineno = 'n/a'
+                    lineno = "n/a"
                 else:
-                    lineno = getattr(node, 'lineno', 1)
+                    lineno = getattr(node, "lineno", 1)
                 yield CodeDocumentComment(
                     filename=filename,
                     lineno=lineno,
-                    name=getattr(node, 'name', None),
-                    text=node_docstring
+                    name=getattr(node, "name", None),
+                    text=node_docstring,
+                    package="",
                 )
         except TypeError:
             # node type may not be able to have docstrings
             pass
 
 
 def iter_extract_docstring_from_lines(
-    lines: Iterable[str]
+    lines: Iterable[str],
 ) -> Iterable[CodeDocumentComment]:
-    return iter_extract_docstring_from_text('\n'.join(lines))
+    return iter_extract_docstring_from_text("\n".join(lines))
 
 
 def iter_extract_docstring_from_file(
-    path: Union[str, Path],
-    encoding: str = DEFAULT_ENCODING
+    path: Union[str, Path], encoding: str = DEFAULT_ENCODING
 ) -> Iterable[CodeDocumentComment]:
     path = Path(path)
     txt = path.read_text(encoding=encoding)
@@ -72,13 +74,11 @@ def iter_extract_docstring_from_file(
                 txt.append(FAKE_FUNC.format(i, "    ".join(c["source"])))
         txt = "\n".join(txt)
         notebook = True
-    return iter_extract_docstring_from_text(txt, filename=str(path),
-                                            notebook=notebook)
+    return iter_extract_docstring_from_text(txt, filename=str(path), notebook=notebook)
 
 
 def iter_extract_docstring_from_files(
-    paths: Iterable[Union[str, Path]],
-    **kwargs
+    paths: Iterable[Union[str, Path]], **kwargs
 ) -> Iterable[CodeDocumentComment]:
     for path in paths:
         yield from iter_extract_docstring_from_file(path, **kwargs)