From e03c20eaaf08b527a8ee8a609daa8ee604b7a28c Mon Sep 17 00:00:00 2001 From: Kareem Zidane Date: Fri, 14 Feb 2020 15:15:33 -0500 Subject: [PATCH 01/64] catching EOF --- lib50/_api.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib50/_api.py b/lib50/_api.py index a1547f6..1788280 100644 --- a/lib50/_api.py +++ b/lib50/_api.py @@ -821,7 +821,7 @@ def _authenticate_ssh(org, repo=None): "Enter passphrase for key", "Permission denied", "Are you sure you want to continue connecting"]) - except pexpect.TIMEOUT: + except (pexpect.EOF, pexpect.TIMEOUT): return None diff --git a/setup.py b/setup.py index 59d3160..6d73751 100644 --- a/setup.py +++ b/setup.py @@ -23,6 +23,6 @@ python_requires=">= 3.6", packages=["lib50"], url="https://github.com/cs50/lib50", - version="2.0.7", + version="2.0.8", include_package_data=True ) From 2f15212504517863369bed9fb56dfb637a5c39f9 Mon Sep 17 00:00:00 2001 From: Chad Sharp Date: Wed, 8 Apr 2020 16:12:07 -0400 Subject: [PATCH 02/64] add honesty prompt support --- lib50/_api.py | 13 +++++++++---- setup.py | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/lib50/_api.py b/lib50/_api.py index 4b809c2..1520b99 100644 --- a/lib50/_api.py +++ b/lib50/_api.py @@ -45,7 +45,7 @@ AUTH_URL = "https://submit.cs50.io" -def push(tool, slug, config_loader, repo=None, data=None, prompt=lambda included, excluded: True): +def push(tool, slug, config_loader, repo=None, data=None, prompt=lambda question, included, excluded: True): """ Push to github.com/org/repo=username/slug if tool exists. Returns username, commit hash @@ -63,13 +63,13 @@ def push(tool, slug, config_loader, repo=None, data=None, prompt=lambda included check_dependencies() # Connect to GitHub and parse the config files - remote, (included, excluded) = connect(slug, config_loader) + remote, (honesty, included, excluded) = connect(slug, config_loader) # Authenticate the user with GitHub, and prepare the submission with authenticate(remote["org"], repo=repo) as user, prepare(tool, slug, user, included): # Show any prompt if specified - if prompt(included, excluded): + if prompt(honesty, included, excluded): username, commit_hash = upload(slug, user, tool, data) format_dict = {"username": username, "slug": slug, "commit_hash": commit_hash} message = remote["message"].format(results=remote["results"].format(**format_dict), **format_dict) @@ -248,6 +248,9 @@ def connect(slug, config_loader): } remote.update(config.get("remote", {})) + honesty = config.get("honesty", _("Keeping in mind the course's policy on " + "academic honesty, are you sure you want to " + "submit these files?")) # Figure out which files to include and exclude included, excluded = files(config.get("files")) @@ -256,7 +259,9 @@ def connect(slug, config_loader): if not included: raise Error(_("No files in this directory are expected for submission.")) - return remote, (included, excluded) + + return remote, (honesty, included, excluded) + @contextlib.contextmanager diff --git a/setup.py b/setup.py index 6d73751..d9de459 100644 --- a/setup.py +++ b/setup.py @@ -23,6 +23,6 @@ python_requires=">= 3.6", packages=["lib50"], url="https://github.com/cs50/lib50", - version="2.0.8", + version="3.0.0", include_package_data=True ) From 2d730580954b311e7304d77fa0da250236f74697 Mon Sep 17 00:00:00 2001 From: Chad Sharp Date: Wed, 8 Apr 2020 16:17:01 -0400 Subject: [PATCH 03/64] fix tests --- tests/lib50_tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/lib50_tests.py b/tests/lib50_tests.py index 3c9c80e..84810de 100644 --- a/tests/lib50_tests.py +++ b/tests/lib50_tests.py @@ -30,7 +30,7 @@ def test_connect(self): f = io.StringIO() open("hello.py", "w").close() with contextlib.redirect_stdout(f): - remote, (included, excluded) = lib50.connect("cs50/lib50/tests/bar", self.loader) + remote, (honesty, included, excluded) = lib50.connect("cs50/lib50/tests/bar", self.loader) self.assertEqual(excluded, set()) self.assertEqual(remote["org"], lib50._api.DEFAULT_PUSH_ORG) @@ -39,7 +39,7 @@ def test_connect(self): loader = lib50.config.Loader("submit50") loader.scope("files", "exclude", "include", "require") with contextlib.redirect_stdout(f): - remote, (included, excluded) = lib50.connect("cs50/lib50/tests/bar", loader) + remote, (honesty, included, excluded) = lib50.connect("cs50/lib50/tests/bar", loader) self.assertEqual(included, {"hello.py"}) def test_missing_problem(self): From cb178791e9a7544ea940e5d6be0ca0aab1535f49 Mon Sep 17 00:00:00 2001 From: Chad Sharp Date: Wed, 8 Apr 2020 16:18:07 -0400 Subject: [PATCH 04/64] check honesty prompt is not false first --- lib50/_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib50/_api.py b/lib50/_api.py index 1520b99..4b7ee00 100644 --- a/lib50/_api.py +++ b/lib50/_api.py @@ -69,7 +69,7 @@ def push(tool, slug, config_loader, repo=None, data=None, prompt=lambda question with authenticate(remote["org"], repo=repo) as user, prepare(tool, slug, user, included): # Show any prompt if specified - if prompt(honesty, included, excluded): + if honesty and prompt(honesty, included, excluded): username, commit_hash = upload(slug, user, tool, data) format_dict = {"username": username, "slug": slug, "commit_hash": commit_hash} message = remote["message"].format(results=remote["results"].format(**format_dict), **format_dict) From 06ad6c2c1a6710af5fc884322b86212ca4c56c2e Mon Sep 17 00:00:00 2001 From: Chad Sharp Date: Wed, 8 Apr 2020 16:19:55 -0400 Subject: [PATCH 05/64] fix comment --- lib50/_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib50/_api.py b/lib50/_api.py index 4b7ee00..ae32266 100644 --- a/lib50/_api.py +++ b/lib50/_api.py @@ -68,7 +68,7 @@ def push(tool, slug, config_loader, repo=None, data=None, prompt=lambda question # Authenticate the user with GitHub, and prepare the submission with authenticate(remote["org"], repo=repo) as user, prepare(tool, slug, user, included): - # Show any prompt if specified + # Show prompt if honesty is not false-y if honesty and prompt(honesty, included, excluded): username, commit_hash = upload(slug, user, tool, data) format_dict = {"username": username, "slug": slug, "commit_hash": commit_hash} From 9fe2c5a26312aa00c21980647c897aadb6b01295 Mon Sep 17 00:00:00 2001 From: Chad Sharp Date: Wed, 8 Apr 2020 16:26:00 -0400 Subject: [PATCH 06/64] Revert "check honesty prompt is not false first" This reverts commit cb178791e9a7544ea940e5d6be0ca0aab1535f49. --- lib50/_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib50/_api.py b/lib50/_api.py index ae32266..1520b99 100644 --- a/lib50/_api.py +++ b/lib50/_api.py @@ -68,8 +68,8 @@ def push(tool, slug, config_loader, repo=None, data=None, prompt=lambda question # Authenticate the user with GitHub, and prepare the submission with authenticate(remote["org"], repo=repo) as user, prepare(tool, slug, user, included): - # Show prompt if honesty is not false-y - if honesty and prompt(honesty, included, excluded): + # Show any prompt if specified + if prompt(honesty, included, excluded): username, commit_hash = upload(slug, user, tool, data) format_dict = {"username": username, "slug": slug, "commit_hash": commit_hash} message = remote["message"].format(results=remote["results"].format(**format_dict), **format_dict) From c5d73b84e8e43368d97f08590c5c8d9a70b15809 Mon Sep 17 00:00:00 2001 From: Chad Sharp Date: Wed, 29 Apr 2020 14:42:07 -0400 Subject: [PATCH 07/64] use True to indicate default --- lib50/_api.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib50/_api.py b/lib50/_api.py index 1520b99..7d4fc50 100644 --- a/lib50/_api.py +++ b/lib50/_api.py @@ -248,9 +248,7 @@ def connect(slug, config_loader): } remote.update(config.get("remote", {})) - honesty = config.get("honesty", _("Keeping in mind the course's policy on " - "academic honesty, are you sure you want to " - "submit these files?")) + honesty = config.get("honesty", True) # Figure out which files to include and exclude included, excluded = files(config.get("files")) From c79c641780e954f07d241e0f68bc9341bbc9b0bb Mon Sep 17 00:00:00 2001 From: Chad Sharp Date: Thu, 14 May 2020 15:33:06 -0400 Subject: [PATCH 08/64] improve _spawn error handling --- lib50/_api.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib50/_api.py b/lib50/_api.py index 4b809c2..53b6e1e 100644 --- a/lib50/_api.py +++ b/lib50/_api.py @@ -593,6 +593,10 @@ def _get_branches(self): return [] else: raise TimeoutError(3) + except Error: + if "Could not resolve host" in child.buffer: + raise TimeoutError(3) + raise # Parse get_refs output for the actual branch names return (line.split()[1].replace("refs/heads/", "") for line in output) From 3573f32a63ec6d7099fa882240b3a21227663eea Mon Sep 17 00:00:00 2001 From: Chad Sharp Date: Thu, 14 May 2020 15:36:44 -0400 Subject: [PATCH 09/64] check buffer and before --- lib50/_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib50/_api.py b/lib50/_api.py index 53b6e1e..a09ee37 100644 --- a/lib50/_api.py +++ b/lib50/_api.py @@ -594,7 +594,7 @@ def _get_branches(self): else: raise TimeoutError(3) except Error: - if "Could not resolve host" in child.buffer: + if "Could not resolve host" in child.before + child.buffer: raise TimeoutError(3) raise From 4202ea96bc3d5fe32dabe839f255be85296af6c3 Mon Sep 17 00:00:00 2001 From: Chad Sharp Date: Thu, 14 May 2020 15:37:51 -0400 Subject: [PATCH 10/64] use ConnectionError instead of TimeoutError --- lib50/_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib50/_api.py b/lib50/_api.py index a09ee37..841733b 100644 --- a/lib50/_api.py +++ b/lib50/_api.py @@ -595,7 +595,7 @@ def _get_branches(self): raise TimeoutError(3) except Error: if "Could not resolve host" in child.before + child.buffer: - raise TimeoutError(3) + raise ConnectionError raise # Parse get_refs output for the actual branch names From 05d6181d1c7826617e06bd802297ca130972b49b Mon Sep 17 00:00:00 2001 From: Chad Sharp Date: Thu, 14 May 2020 15:39:47 -0400 Subject: [PATCH 11/64] reraise ConnectionError --- lib50/_api.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib50/_api.py b/lib50/_api.py index 841733b..10848fa 100644 --- a/lib50/_api.py +++ b/lib50/_api.py @@ -554,6 +554,8 @@ def __init__(self, slug, offline=False, github_token=None): if not offline: raise ConnectionError("Could not connect to GitHub, it seems you are offline.") branches = [] + except ConnectionError: + raise except Error: branches = [] From f60b436f152fa742b2fff8f2f604451d082da2dd Mon Sep 17 00:00:00 2001 From: jelleas Date: Mon, 1 Jun 2020 15:50:09 +0200 Subject: [PATCH 12/64] initial docs --- .gitignore | 2 + docs/Makefile | 20 +++++ docs/requirements.txt | 3 + docs/source/api.rst | 13 ++++ docs/source/conf.py | 166 ++++++++++++++++++++++++++++++++++++++++++ docs/source/index.rst | 54 ++++++++++++++ lib50/_api.py | 20 ++++- 7 files changed, 276 insertions(+), 2 deletions(-) create mode 100644 docs/Makefile create mode 100644 docs/requirements.txt create mode 100644 docs/source/api.rst create mode 100644 docs/source/conf.py create mode 100644 docs/source/index.rst diff --git a/.gitignore b/.gitignore index 894a44c..5815030 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +.DS_store + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..445d1c2 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = lib50 +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..04c8577 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,3 @@ +sphinx +sphinx-autobuild +sphinx_rtd_theme diff --git a/docs/source/api.rst b/docs/source/api.rst new file mode 100644 index 0000000..3eb4f27 --- /dev/null +++ b/docs/source/api.rst @@ -0,0 +1,13 @@ +.. _api: + +API docs +======== + +.. _check50: + +lib50 +******* + +.. automodule:: lib50 + :members: + :imported-members: diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000..89527cf --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# lib50 documentation build configuration file, created by +# sphinx-quickstart on Wed Jun 27 13:24:24 2018. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ['sphinx.ext.autodoc'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = 'lib50' +copyright = '2018, CS50' +author = 'CS50' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '' +# The full version, including alpha/beta/rc tags. +release = '' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = [] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = "sphinx_rtd_theme" + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +html_theme_options = {"collapse_navigation" : False} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = [] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# This is required for the alabaster theme +# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars +html_sidebars = { + '**': [ + 'relations.html', # needs 'show_related': True theme option to display + 'searchbox.html', + ] +} + + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = 'lib50doc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'lib50.tex', 'lib50 Documentation', + 'CS50', 'manual'), +] + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'lib50', 'lib50 Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'lib50', 'lib50 Documentation', + author, 'lib50', 'One line description of project.', + 'Miscellaneous'), +] diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000..de45443 --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,54 @@ +``lib50`` +=========== + +.. toctree:: + :hidden: + :maxdepth: 3 + :caption: Contents: + + api + +.. Indices and tables +.. ================== + +.. * :ref:`genindex` +.. * :ref:`api` +.. * :ref:`modindex` +.. * :ref:`search` + + +lib50 is CS50's library for common functionality shared between its tools. The library is, like most of CS50's projects, open-source, but its intention is to serve as an internal library for CS50's own tools. As such it is our current recommendation to not use lib50 as a dependency of one's own projects. + +To promote reuse of functionality across CS50 tools, lib50 is designed to be tool agnostic. lib50 provides just the core functionality, but the semantics of that functionality are left up to the tool. For instance, submit50 adds the notion of a submission to a push to GitHub, whereas it is lib50 that provides the ``push`` function that ultimately handles the workflow with GitHub. Or per another example, lib50 provides the functionality to parse and validate ``.cs50.yml`` configuration files, but each individual tool (check50, submit50 and lab50) specifies their own options and handles their own logic. + +When looking for a piece of functionality that exists in other CS50 tools, odds are it lives in lib50. Increasingly so then, does the following apply: + +"If it seems useful, it probably already exists in lib50" - Chad Sharp, 2019. + + +Installation +************ + +First make sure you have Python 3.6 or higher installed. You can download Python |download_python|. + +.. |download_python| raw:: html + + here + +lib50 has a dependency on git, please make sure to |install_git| if git is not already installed. + +.. |install_git| raw:: html + + install git + +To install lib50 under Linux / OS X: + +.. code-block:: bash + + pip install lib50 + +Under Windows, please |install_windows_sub|. Then install lib50 within the subsystem. + +.. |install_windows_sub| raw:: html + + install the Linux subsystem diff --git a/lib50/_api.py b/lib50/_api.py index ef0a3d8..05f2177 100644 --- a/lib50/_api.py +++ b/lib50/_api.py @@ -47,8 +47,24 @@ def push(tool, slug, config_loader, repo=None, data=None, prompt=lambda question, included, excluded: True): """ - Push to github.com/org/repo=username/slug if tool exists. - Returns username, commit hash + Pushes to Github in name of a tool. + What should be pushed is configured by the tool and its configuration in the .cs50.yml file identified by the slug. + By default, this function pushes to https://github.com/org=me50/repo=/branch=. + + :param tool: name of the tool that initialized the push + :type tool: str + :param slug: the slug identifying a .cs50.yml config file in a GitHub repo. This slug is also the branch in the student's repo to which this will push. + :type slug: str + :param config_loader: a config loader for the tool, being able to parse the .cs50.yml config file for the tool. + :type config_loader: lib50.config.ConfigLoader + :param repo: an alternative repo to push to, otherwise the default is used: github.com/me50/ + :type repo: str, optional + :param data: key value pairs that end up in the commit message. This can be used to communicate data with a backend. + :type data: dict of strings, optional + :param prompt: a prompt shown just before the push. In case this prompt returns false, the push is aborted. This lambda function has access to an honesty prompt configured in .cs50,yml, and all files that will be included and excluded in the push. + :type prompt: lambda str, list, list => bool, optional + :return: GitHub username and the commit hash + :type: tuple(str, str) """ if data is None: From 5601a3c84a1c8aae8d4394f6a2e0bf668d5c054d Mon Sep 17 00:00:00 2001 From: jelleas Date: Mon, 1 Jun 2020 16:36:33 +0200 Subject: [PATCH 13/64] local() docs --- lib50/_api.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/lib50/_api.py b/lib50/_api.py index 05f2177..51f175f 100644 --- a/lib50/_api.py +++ b/lib50/_api.py @@ -96,8 +96,18 @@ def push(tool, slug, config_loader, repo=None, data=None, prompt=lambda question def local(slug, offline=False, remove_origin=False, github_token=None): """ - Create/update local copy of github.com/org/repo/branch. - Returns path to local copy + Create/update local copy of the GitHub repo indentified by slug. + + :param slug: the slug identifying a GitHub repo. + :type slug: str + :param offline: a flag that indicates whether the user is offline. If so, then the local copy is only checked, but not updated. + :type offline: bool, optional + :param remove_origin: a flag, that when set to True, will remove origin as a remote of the git repo. + :type remove_origin: bool, optional + :param github_token: a GitHub authentication token used to verify the slug, only needed if the slug identifies a private repo. + :type github_token: str, optional + :return: path to local copy + :type: pathlib.Path """ # Parse slug From f1c19426b1260cc3a02656fb9433573527c12604 Mon Sep 17 00:00:00 2001 From: jelleas Date: Mon, 1 Jun 2020 16:49:10 +0200 Subject: [PATCH 14/64] working_area docs --- lib50/_api.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/lib50/_api.py b/lib50/_api.py index 51f175f..ea89665 100644 --- a/lib50/_api.py +++ b/lib50/_api.py @@ -148,9 +148,20 @@ def local(slug, offline=False, remove_origin=False, github_token=None): @contextlib.contextmanager def working_area(files, name=""): """ - Copy all files to a temporary directory (the working area) - Optionally names the working area name - Returns path to the working area + A contextmanager that copies all files to a temporary directory (the working area) + + :param files: all files to copy to the temporary directory + :type files: list of string(s) or pathlib.Path(s) + :param name: name of the temporary directory + :type name: str, optional + :return: path to the working area + :type: pathlib.Path + + Example usage:: + + with working_area(["foo.c", "bar.py"], name="baz") as area: + print(list(area.glob("**/*"))) + """ with tempfile.TemporaryDirectory() as dir: dir = Path(Path(dir) / name) From 44d5a2a50423ff1186cc06a20c73a735cdb3044b Mon Sep 17 00:00:00 2001 From: jelleas Date: Mon, 1 Jun 2020 17:14:28 +0200 Subject: [PATCH 15/64] cd & files docs --- lib50/_api.py | 56 ++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 49 insertions(+), 7 deletions(-) diff --git a/lib50/_api.py b/lib50/_api.py index ea89665..3959fd0 100644 --- a/lib50/_api.py +++ b/lib50/_api.py @@ -176,7 +176,21 @@ def working_area(files, name=""): @contextlib.contextmanager def cd(dest): - """ Temporarily cd into a directory""" + """ + A contextmanager for temporarily changing directory. + + :param dest: the path to the directory + :type dest: str or pathlib.Path + :return: dest unchanged + :type: str or pathlib.Path + + Example usage:: + + import os + with cd("foo") as current_dir: + print(os.getcwd()) + + """ origin = os.getcwd() try: os.chdir(dest) @@ -191,12 +205,40 @@ def files(patterns, exclude_tags=("exclude",), root="."): """ - Takes a list of lib50._config.TaggedValue returns which files should be included and excluded from `root`. - Any pattern tagged with a tag - from include_tags will be included - from require_tags can only be a file, that will then be included. MissingFilesError is raised if missing - from exclude_tags will be excluded - Any pattern in always_exclude will always be excluded. + Based on a list of patterns (lib50.config.TaggedValue) determine which files should be included and excluded. + Any pattern tagged with a tag: + + * from ``include_tags`` will be included + * from ``require_tags`` can only be a file, that will then be included. ``MissingFilesError`` is raised if missing. + * from ``exclude_tags`` will be excluded + + :param patterns: patterns that are processed in order to determine which files should be included and excluded. + :type patterns: list of lib50.config.TaggedValue + :param require_tags: tags that identify a file as required and through that included + :type require_tags: list of strings, optional + :param include_tags: tags that identify a pattern as included + :type include_tags: list of strings, optional + :param exclude_tags: tags that identify a pattern as excluded + :type exclude_tags: list of strings, optional + :param root: the root directory from which to look for files. Defaults to the current directory. + :type root: str or pathlib.Path, optional + :return: all included files and all excluded files + :type: tuple(set of strings, set of strings) + + Example usage:: + + from lib50.config import TaggedValue + + open("foo.py", "w").close() + open("bar.c", "w").close() + open("baz.h", "w").close() + + patterns = [TaggedValue("*", "exclude"), + TaggedValue("*.c", "include"), + TaggedValue("baz.h", "require")] + + print(files(patterns)) # prints ({'bar.c', 'baz.h'}, {'foo.py'}) + """ require_tags = list(require_tags) include_tags = list(include_tags) From 1b4ffde7f13fa61cbeaa1b74861aa8a4176cdd78 Mon Sep 17 00:00:00 2001 From: jelleas Date: Mon, 1 Jun 2020 17:33:09 +0200 Subject: [PATCH 16/64] identify => mark --- lib50/_api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib50/_api.py b/lib50/_api.py index 3959fd0..bb301b7 100644 --- a/lib50/_api.py +++ b/lib50/_api.py @@ -214,11 +214,11 @@ def files(patterns, :param patterns: patterns that are processed in order to determine which files should be included and excluded. :type patterns: list of lib50.config.TaggedValue - :param require_tags: tags that identify a file as required and through that included + :param require_tags: tags that mark a file as required and through that included :type require_tags: list of strings, optional - :param include_tags: tags that identify a pattern as included + :param include_tags: tags that mark a pattern as included :type include_tags: list of strings, optional - :param exclude_tags: tags that identify a pattern as excluded + :param exclude_tags: tags that mark a pattern as excluded :type exclude_tags: list of strings, optional :param root: the root directory from which to look for files. Defaults to the current directory. :type root: str or pathlib.Path, optional From 473f0cb966f8436d6f2bd41209c0e46ee4415ae9 Mon Sep 17 00:00:00 2001 From: jelleas Date: Mon, 1 Jun 2020 17:49:51 +0200 Subject: [PATCH 17/64] connect() docs --- lib50/_api.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/lib50/_api.py b/lib50/_api.py index bb301b7..82f6c5d 100644 --- a/lib50/_api.py +++ b/lib50/_api.py @@ -55,7 +55,7 @@ def push(tool, slug, config_loader, repo=None, data=None, prompt=lambda question :type tool: str :param slug: the slug identifying a .cs50.yml config file in a GitHub repo. This slug is also the branch in the student's repo to which this will push. :type slug: str - :param config_loader: a config loader for the tool, being able to parse the .cs50.yml config file for the tool. + :param config_loader: a config loader for the tool that is able to parse the .cs50.yml config file for the tool. :type config_loader: lib50.config.ConfigLoader :param repo: an alternative repo to push to, otherwise the default is used: github.com/me50/ :type repo: str, optional @@ -212,7 +212,7 @@ def files(patterns, * from ``require_tags`` can only be a file, that will then be included. ``MissingFilesError`` is raised if missing. * from ``exclude_tags`` will be excluded - :param patterns: patterns that are processed in order to determine which files should be included and excluded. + :param patterns: patterns that are processed in order, to determine which files should be included and excluded. :type patterns: list of lib50.config.TaggedValue :param require_tags: tags that mark a file as required and through that included :type require_tags: list of strings, optional @@ -300,9 +300,25 @@ def files(patterns, def connect(slug, config_loader): """ - Ensure .cs50.yaml and tool key exists, raises Error otherwise - Check that all required files as per .cs50.yaml are present - Returns org, and a tuple of included and excluded files + Connects to a GitHub repo indentified by slug. + Then parses the ``.cs50.yml`` config file with the ``config_loader``. + If not all required files are present, per the ``files`` tag in ``.cs50.yml``, an ``Error`` is raised. + + :param slug: the slug identifying a GitHub repo. + :type slug: str + :param config_loader: a config loader that is able to parse the .cs50.yml config file for a tool. + :type config_loader: lib50.config.ConfigLoader + :return: the remote configuration (org, message, callback, results), and the input for a prompt (honesty question, included files, excluded files) + :type: tuple(dict, tuple(str, set, set)) + + Example usage:: + + import submit50 + + open("hello.c", "w").close() + + remote, (honesty, included, excluded) = connect("cs50/problems/2019/x/hello", submit50.CONFIG_LOADER) + """ with ProgressBar(_("Connecting")): # Get the config from GitHub at slug From 9e3f583f6d6b0c25f250c61b92c37647f92323bc Mon Sep 17 00:00:00 2001 From: jelleas Date: Mon, 1 Jun 2020 17:55:40 +0200 Subject: [PATCH 18/64] add lib50 import to ensure that examples can be copy-pasted --- lib50/_api.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib50/_api.py b/lib50/_api.py index 82f6c5d..627942e 100644 --- a/lib50/_api.py +++ b/lib50/_api.py @@ -159,6 +159,8 @@ def working_area(files, name=""): Example usage:: + from lib50 import working_area + with working_area(["foo.c", "bar.py"], name="baz") as area: print(list(area.glob("**/*"))) @@ -186,7 +188,9 @@ def cd(dest): Example usage:: + from lib50 import cd import os + with cd("foo") as current_dir: print(os.getcwd()) @@ -227,6 +231,7 @@ def files(patterns, Example usage:: + from lib50 import files from lib50.config import TaggedValue open("foo.py", "w").close() @@ -313,6 +318,7 @@ def connect(slug, config_loader): Example usage:: + from lib50 import connect import submit50 open("hello.c", "w").close() From a903e33373d25c3a531bc8e40cda6cd755e45cc3 Mon Sep 17 00:00:00 2001 From: jelleas Date: Tue, 2 Jun 2020 14:29:20 +0200 Subject: [PATCH 19/64] connect, prepare, upload docs --- lib50/_api.py | 80 +++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 68 insertions(+), 12 deletions(-) diff --git a/lib50/_api.py b/lib50/_api.py index 627942e..8f09b8e 100644 --- a/lib50/_api.py +++ b/lib50/_api.py @@ -362,13 +362,25 @@ def connect(slug, config_loader): return remote, (honesty, included, excluded) - @contextlib.contextmanager def authenticate(org, repo=None): """ - Authenticate with GitHub via SSH if possible - Otherwise authenticate via HTTPS - Returns an authenticated User + A contextmanager that authenticates a user with GitHub via SSH if possible, otherwise via HTTPS. + + :param org: GitHub organisation to authenticate with + :type org: str + :param repo: GitHub repo (part of the org) to authenticate with. Default is the user's GitHub login. + :type repo: str, optional + :return: an authenticated user + :type: lib50.User + + Example usage:: + + from lib50 import authenticate + + with authenticate("me50") as user: + print(user.name) + """ with ProgressBar(_("Authenticating")) as progress_bar: user = _authenticate_ssh(org, repo=repo) @@ -384,12 +396,35 @@ def authenticate(org, repo=None): @contextlib.contextmanager def prepare(tool, branch, user, included): """ - Prepare git for pushing - Check that there are no permission errors - Add necessities to git config - Stage files - Stage files via lfs if necessary - Check that atleast one file is staged + A contextmanager that prepares git for pushing: + + * Check that there are no permission errors + * Add necessities to git config + * Stage files + * Stage files via lfs if necessary + * Check that atleast one file is staged + + :param tool: name of the tool that started the push + :type tool: str + :param branch: git branch to switch to + :type branch: str + :param user: the user who has access to the repo, and will ultimately author a commit + :type user: lib50.User + :param included: a list of files that are to be staged in git + :type included: list of string(s) or pathlib.Path(s) + :return: None + :type: None + + Example usage:: + + from lib50 import authenticate, prepare, upload + + with authenticate("me50") as user: + tool = "submit50" + branch = "cs50/problems/2019/x/hello" + with prepare(tool, branch, user, ["hello.c"]): + upload(branch, user, tool, {tool:True}) + """ with working_area(included) as area: with ProgressBar(_("Verifying")): @@ -442,8 +477,29 @@ def prepare(tool, branch, user, included): def upload(branch, user, tool, data): """ - Commit + push to branch - Returns username, commit hash + Commit + push to a branch + + :param branch: git branch to commit and push to + :type branch: str + :param user: authenticated user who can push to the repo and branch + :type user: lib50.User + :param tool: name of the tool that started the push + :type tool: str + :param data: key value pairs that end up in the commit message. This can be used to communicate data with a backend. + :type data: dict of strings + :return: username and commit hash + :type: tuple(str, str) + + Example usage:: + + from lib50 import authenticate, prepare, upload + + with authenticate("me50") as user: + tool = "submit50" + branch = "cs50/problems/2019/x/hello" + with prepare(tool, branch, user, ["hello.c"]): + upload(branch, user, tool, {tool:True}) + """ with ProgressBar(_("Uploading")): From 58efb9003e317af812ae55bba93c142c37954768 Mon Sep 17 00:00:00 2001 From: jelleas Date: Tue, 2 Jun 2020 14:42:45 +0200 Subject: [PATCH 20/64] fetch_config + get_local_slugs docs --- lib50/_api.py | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/lib50/_api.py b/lib50/_api.py index 8f09b8e..fc9b149 100644 --- a/lib50/_api.py +++ b/lib50/_api.py @@ -520,8 +520,20 @@ def upload(branch, user, tool, data): def fetch_config(slug): """ Fetch the config file at slug from GitHub. - Returns the unparsed json as a string. - Raises InvalidSlugError if there is no config file at slug. + + :param slug: a slug identifying a location on GitHub to fetch the config from. + :type slug: str + :return: the config in the form of unparsed json + :type: str + :raises lib50.InvalidSlugError: if there is no config file at slug. + + Example usage:: + + from lib50 import fetch_config + + config = fetch_config("cs50/problems/2019/x/hello") + print(config) + """ # Parse slug slug = Slug(slug) @@ -555,8 +567,23 @@ def fetch_config(slug): def get_local_slugs(tool, similar_to=""): """ - Get all slugs for tool of lib50 has a local copy. - If similar_to is given, ranks local slugs by similarity to similar_to. + Get all slugs for tool of which lib50 has a local copy. + If similar_to is given, ranks and sorts local slugs by similarity to similar_to. + + :param tool: tool for which to get the local slugs + :type tool: str + :param similar_to: ranks and sorts local slugs by similarity to this slug + :type similar_to: str, optional + :return: list of slugs + :type: list of strings + + Example usage:: + + from lib50 import get_local_slugs + + slugs = get_local_slugs("check50", similar_to="cs50/problems/2019/x/hllo") + print(slugs) + """ # Extract org and repo from slug to limit search similar_to = similar_to.strip("/") From b278d6c4e51b5ed95196bea91b32a952cab3a80f Mon Sep 17 00:00:00 2001 From: jelleas Date: Tue, 2 Jun 2020 15:12:31 +0200 Subject: [PATCH 21/64] Errors and ProgressBar docs --- docs/source/conf.py | 9 +++++++++ lib50/_api.py | 28 +++++++++++++++++++++++++--- lib50/_errors.py | 18 ++++++++++++++++++ 3 files changed, 52 insertions(+), 3 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 89527cf..7127918 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -164,3 +164,12 @@ author, 'lib50', 'One line description of project.', 'Miscellaneous'), ] + +# Source: https://stackoverflow.com/questions/5599254/how-to-use-sphinxs-autodoc-to-document-a-classs-init-self-method +def skip(app, what, name, obj, would_skip, options): + if name == "__init__": + return False + return would_skip + +def setup(app): + app.connect("autodoc-skip-member", skip) diff --git a/lib50/_api.py b/lib50/_api.py index fc9b149..76ddcef 100644 --- a/lib50/_api.py +++ b/lib50/_api.py @@ -805,11 +805,30 @@ def __str__(self): class ProgressBar: - """Show a progress bar starting with message.""" + """ + A contextmanager that shows a progress bar starting with message. + + Example usage:: + + from lib50 import ProgressBar + import time + + with ProgressBar("uploading") as bar: + time.sleep(5) + bar.stop() + time.sleep(5) + + """ DISABLED = False TICKS_PER_SECOND = 2 def __init__(self, message, output_stream=None): + """ + :param message: the message of the progress bar, what the user is waiting on + :type message: str + :param output_stream: a stream to write the progress bar to + :type output_stream: a stream or file-like object + """ if output_stream is None: output_stream = sys.stderr @@ -948,10 +967,13 @@ def get_content(org, repo, branch, filepath): def check_github_status(): """ - Pings the githubstatus API. Raises an Error if the Git Operations and/or + Pings the githubstatus API. Raises a ConnectionError if the Git Operations and/or API requests components show an increase in errors. - """ + :return: None + :type: None + :raises lib50.ConnectionError: if the Git Operations and/or API requests components show an increase in errors. + """ # https://www.githubstatus.com/api status_result = requests.get("https://kctbh9vrtdwd.statuspage.io/api/v2/components.json") diff --git a/lib50/_errors.py b/lib50/_errors.py index 98933d6..9e454a8 100644 --- a/lib50/_errors.py +++ b/lib50/_errors.py @@ -4,15 +4,28 @@ __all__ = ["Error", "InvalidSlugError", "MissingFilesError", "InvalidConfigError", "MissingToolError", "TimeoutError", "ConnectionError"] class Error(Exception): + """A generic lib50 Error. lib50 Errors can carry arbitrary data in the dict ``Error.payload``,""" def __init__(self, *args, **kwargs): + """""" super().__init__(*args, **kwargs) self.payload = {} class InvalidSlugError(Error): + """A ``lib50.Error`` signalling that a slug is invalid.""" pass class MissingFilesError(Error): + """ + A ``ib50.Error`` signalling that files are missing. + This error's payload has a ``files`` and ``dir`` key. + ``MissingFilesError.payload["files"]`` are all the missing files in a list of strings. + ``MissingFilesError.payload["dir"]`` is the current working directory (cwd) from when this error was raised. + """ def __init__(self, files): + """ + :param files: the missing files that caused the error + :type files: list of string(s) or Pathlib.path(s) + """ cwd = os.getcwd().replace(os.path.expanduser("~"), "~", 1) super().__init__("{}\n{}\n{}".format( _("You seem to be missing these required files:"), @@ -23,16 +36,21 @@ def __init__(self, files): class InvalidConfigError(Error): + """A ``lib50.Error`` signalling that a config is invalid.""" pass class MissingToolError(InvalidConfigError): + """A more specific ``lib50.InvalidConfigError`` signalling that an entry for a tool is missing in the config.""" pass class TimeoutError(Error): + """A ``lib50.Error`` signalling a timeout has occured.""" pass class ConnectionError(Error): + """A ``lib50.Error`` signalling a connection has errored.""" pass class InvalidSignatureError(Error): + """A ``lib50.Error`` signalling the signature of a payload is invalid.""" pass From 4f094cc66abcb37bf340ce4996247bcfe486e5d8 Mon Sep 17 00:00:00 2001 From: jelleas Date: Tue, 2 Jun 2020 15:18:54 +0200 Subject: [PATCH 22/64] rm mention of Submission --- lib50/_api.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib50/_api.py b/lib50/_api.py index 76ddcef..845c81e 100644 --- a/lib50/_api.py +++ b/lib50/_api.py @@ -315,6 +315,8 @@ def connect(slug, config_loader): :type config_loader: lib50.config.ConfigLoader :return: the remote configuration (org, message, callback, results), and the input for a prompt (honesty question, included files, excluded files) :type: tuple(dict, tuple(str, set, set)) + :raises lib50.InvalidSlugError: if the slug is invalid for the tool + :raises lib50.Error: if no files are staged. For instance the slug expects .c files, but there are only .py files present. Example usage:: @@ -356,7 +358,7 @@ def connect(slug, config_loader): # Check that at least 1 file is staged if not included: - raise Error(_("No files in this directory are expected for submission.")) + raise Error(_("No files in this directory are expected by {}.".format(slug))) return remote, (honesty, included, excluded) From d7f274a09c92c0ae22ee9f710c64a0c7e3d5020b Mon Sep 17 00:00:00 2001 From: jelleas Date: Tue, 2 Jun 2020 17:38:11 +0200 Subject: [PATCH 23/64] examples are good, lets do more of those --- lib50/_api.py | 60 ++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 57 insertions(+), 3 deletions(-) diff --git a/lib50/_api.py b/lib50/_api.py index 845c81e..a3088e9 100644 --- a/lib50/_api.py +++ b/lib50/_api.py @@ -51,6 +51,8 @@ def push(tool, slug, config_loader, repo=None, data=None, prompt=lambda question What should be pushed is configured by the tool and its configuration in the .cs50.yml file identified by the slug. By default, this function pushes to https://github.com/org=me50/repo=/branch=. + ``lib50.push`` executes the workflow: ``lib50.connect``, ``lib50.authenticate``, ``lib50.prepare`` and ``lib50.upload``. + :param tool: name of the tool that initialized the push :type tool: str :param slug: the slug identifying a .cs50.yml config file in a GitHub repo. This slug is also the branch in the student's repo to which this will push. @@ -65,8 +67,17 @@ def push(tool, slug, config_loader, repo=None, data=None, prompt=lambda question :type prompt: lambda str, list, list => bool, optional :return: GitHub username and the commit hash :type: tuple(str, str) - """ + Example usage:: + + from lib50 import push + import submit50 + + name, hash = push("submit50", "cs50/problems/2019/x/hello", submit50.CONFIG_LOADER) + print(name) + print(hash) + + """ if data is None: data = {} @@ -108,6 +119,14 @@ def local(slug, offline=False, remove_origin=False, github_token=None): :type github_token: str, optional :return: path to local copy :type: pathlib.Path + + Example usage:: + + from lib50 import local + + path = local("cs50/problems/2019/x/hello") + print(list(path.glob("**/*"))) + """ # Parse slug @@ -425,7 +444,7 @@ def prepare(tool, branch, user, included): tool = "submit50" branch = "cs50/problems/2019/x/hello" with prepare(tool, branch, user, ["hello.c"]): - upload(branch, user, tool, {tool:True}) + upload(branch, user, tool, {}) """ with working_area(included) as area: @@ -672,20 +691,33 @@ def check_dependencies(): def logout(): + """Log out from git.""" _run(f"git credential-cache --socket {_CREDENTIAL_SOCKET} exit") @attr.s(slots=True) class User: + """An authenticated GitHub user that has write access to org/repo.""" name = attr.ib() repo = attr.ib() org = attr.ib() email = attr.ib(default=attr.Factory(lambda self: f"{self.name}@users.noreply.github.com", takes_self=True), - init=False) + init=False) class Git: + """ + A stateful helper class for formatting git commands. + + To avoid confusion, and because these are not directly relevant to users, + the class variables ``cache`` and ``working_area`` are excluded from logs. + + Example usage:: + + command = Git().set("-C {folder}", folder="foo")("git clone {repo}", repo="foo") + print(command) + """ cache = "" working_area = "" @@ -716,6 +748,25 @@ def __call__(self, command, **format_args): class Slug: + """ + A CS50 slug that uniquely identifies a location on GitHub. + + A slug is formatted as follows: /// + Both the branch and the problem can have an arbitrary number of slashes. + ``lib50.Slug`` performs validation on the slug, by querrying GitHub, + pulling in all branches, and then by finding a branch and problem that matches the slug. + + Example usage:: + + from lib50._api import Slug + + slug = Slug("cs50/problems/2019/x/hello") + print(slug.org) + print(slug.repo) + print(slug.branch) + print(slug.problem) + + """ def __init__(self, slug, offline=False, github_token=None): """Parse /// from slug.""" self.slug = self.normalize_case(slug) @@ -794,6 +845,7 @@ def _get_branches(self): @staticmethod def normalize_case(slug): + """Normalize the case of a slug in string form""" parts = slug.split("/") if len(parts) < 3: raise InvalidSlugError(_("Invalid slug")) @@ -884,6 +936,7 @@ def flush(self): @contextlib.contextmanager def _spawn(command, quiet=False, timeout=None): + """Run (spawn) a command with `pexpect.spawn`""" # Spawn command child = pexpect.spawn( command, @@ -944,6 +997,7 @@ def _glob(pattern, skip_dirs=False): def _match_files(universe, pattern): + """From a universe of files, get just those files that match the pattern.""" # Implicit recursive iff no / in pattern and starts with * if "/" not in pattern and pattern.startswith("*"): pattern = f"**/{pattern}" From bd4260ddd8e555810555f47e7cd410ccd2534b9b Mon Sep 17 00:00:00 2001 From: jelleas Date: Tue, 2 Jun 2020 17:47:07 +0200 Subject: [PATCH 24/64] setup autodoc for lib50.config lib50.crypto --- docs/source/api.rst | 16 ++++++++++++++-- lib50/_api.py | 9 +++++++-- lib50/config.py | 9 +++++---- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/docs/source/api.rst b/docs/source/api.rst index 3eb4f27..8c3bd80 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -9,5 +9,17 @@ lib50 ******* .. automodule:: lib50 - :members: - :imported-members: + :members: + :imported-members: + +lib50.config +************ + +.. automodule:: lib50.config + :members: + +lib50.crypto +************ + +.. automodule:: lib50.crypto + :members: diff --git a/lib50/_api.py b/lib50/_api.py index a3088e9..393b6be 100644 --- a/lib50/_api.py +++ b/lib50/_api.py @@ -126,7 +126,7 @@ def local(slug, offline=False, remove_origin=False, github_token=None): path = local("cs50/problems/2019/x/hello") print(list(path.glob("**/*"))) - + """ # Parse slug @@ -691,7 +691,12 @@ def check_dependencies(): def logout(): - """Log out from git.""" + """ + Log out from git. + + :return: None + :type: None + """ _run(f"git credential-cache --socket {_CREDENTIAL_SOCKET} exit") diff --git a/lib50/config.py b/lib50/config.py index c8f7218..5949e82 100644 --- a/lib50/config.py +++ b/lib50/config.py @@ -16,10 +16,11 @@ def get_config_filepath(path): """ Looks for the following files in order at path: - - .cs50.yaml - - .cs50.yml - If either exists, - returns path to that file (i.e. /.cs50.yaml or /.cs50.yml) + + * .cs50.yaml + * .cs50.yml + + If either exists, returns path to that file (i.e. /.cs50.yaml or /.cs50.yml) Raises errors.Error otherwise. """ path = pathlib.Path(path) From a1de2cd97db4f630711470a3753f24a216ab1de2 Mon Sep 17 00:00:00 2001 From: jelleas Date: Tue, 2 Jun 2020 18:31:15 +0200 Subject: [PATCH 25/64] lib50.config docs --- lib50/_api.py | 4 +- lib50/config.py | 110 ++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 103 insertions(+), 11 deletions(-) diff --git a/lib50/_api.py b/lib50/_api.py index 393b6be..b41c8b9 100644 --- a/lib50/_api.py +++ b/lib50/_api.py @@ -58,7 +58,7 @@ def push(tool, slug, config_loader, repo=None, data=None, prompt=lambda question :param slug: the slug identifying a .cs50.yml config file in a GitHub repo. This slug is also the branch in the student's repo to which this will push. :type slug: str :param config_loader: a config loader for the tool that is able to parse the .cs50.yml config file for the tool. - :type config_loader: lib50.config.ConfigLoader + :type config_loader: lib50.config.Loader :param repo: an alternative repo to push to, otherwise the default is used: github.com/me50/ :type repo: str, optional :param data: key value pairs that end up in the commit message. This can be used to communicate data with a backend. @@ -331,7 +331,7 @@ def connect(slug, config_loader): :param slug: the slug identifying a GitHub repo. :type slug: str :param config_loader: a config loader that is able to parse the .cs50.yml config file for a tool. - :type config_loader: lib50.config.ConfigLoader + :type config_loader: lib50.config.Loader :return: the remote configuration (org, message, callback, results), and the input for a prompt (honesty question, included files, excluded files) :type: tuple(dict, tuple(str, set, set)) :raises lib50.InvalidSlugError: if the slug is invalid for the tool diff --git a/lib50/config.py b/lib50/config.py index 5949e82..753ebf4 100644 --- a/lib50/config.py +++ b/lib50/config.py @@ -1,3 +1,5 @@ +"""An API for retrieving and parsing ``.cs50.yml`` configs.""" + import collections import enum @@ -17,11 +19,27 @@ def get_config_filepath(path): """ Looks for the following files in order at path: - * .cs50.yaml - * .cs50.yml + * ``.cs50.yaml`` + * ``.cs50.yml`` + + If only one exists, returns path to that file (i.e. ``/.cs50.yaml`` or ``/.cs50.yml``) + Raises ``lib50.Error`` otherwise. + + :param path: local path to a ``.cs50.yml`` config file + :type path: str or pathlib.Path + :return: path to the config file + :type: pathlib.Path + :raises lib50.Error: if zero or more than one config files exist + + Example usage:: + + from lib50 import local + from lib50.config import get_config_filepath + + path = local("cs50/problems/2019/x/hello") + config_path = get_config_filepath(path) + print(config_path) - If either exists, returns path to that file (i.e. /.cs50.yaml or /.cs50.yml) - Raises errors.Error otherwise. """ path = pathlib.Path(path) @@ -38,8 +56,14 @@ def get_config_filepath(path): class TaggedValue: - """A value tagged in a .yaml file""" + """A value tagged in a ``.yml`` file""" def __init__(self, value, tag): + """ + :param value: the tagged value + :type value: str + :param tag: the yaml tag, with or without the syntactically required ``!`` + :type tag: str + """ self.value = value self.tag = tag[1:] if tag.startswith("!") else tag @@ -48,9 +72,45 @@ def __repr__(self): class Loader: - class _TaggedYamlValue: - """A value tagged in a .yaml file""" + """ + A config loader (parser) that can parse a tools section of ``.cs50.yml`` config files. + + The loader can be configured to parse and validate custom yaml tags. + These tags can be global, in which case they can occur anywhere in a tool's section. + Or scoped to a top level key within a tool's section, in which case these tags can only occur there. + + Tags can also have defaults. + In which case there does not need to be a value next to the tag in the config file. + + Example usage:: + + from lib50 import local + from lib50.config import Loader, get_config_filepath + + # Get a local config file + path = local("cs50/problems/2019/x/hello") + config_path = get_config_filepath(path) + # Create a loader for the tool submit50 + loader = Loader("submit50") + + # Allow the tags include/exclude/require to exist only within the files key of submit50 + loader.scope("files", "include", "exclude", "require") + + # Load, parse and validate the config file + with open(config_path) as f: + config = loader.load(f.read()) + + print(config) + + """ + + class _TaggedYamlValue: + """ + A value tagged in a .yaml file. + This is effectively an extension of TaggedValue that keeps track of all possible tags. + This only exists for purposes of validation within ``Loader``. + """ def __init__(self, value, tag, *tags): """ value - the actual value @@ -74,13 +134,33 @@ def __repr__(self): def __init__(self, tool, *global_tags, default=None): + """ + :param tool: the tool for which to load + :type tool: str + :param global_tags: any tags that can be applied globally (within the tool's section) + :type global_tags: str + :param default: a default value for global tags + :type default: anything, optional + """ self._global_tags = self._ensure_exclamation(global_tags) self._global_default = default if not default or default.startswith("!") else "!" + default self._scopes = collections.defaultdict(list) self.tool = tool def scope(self, key, *tags, default=None): - """Only apply tags and default for top-level key, effectively scoping the tags.""" + """ + Only apply tags and default for top-level key of the tool's section. + This effectively scopes the tags to just that top-level key. + + :param key: the top-level key + :type key: str + :param tags: any tags that can be applied within the top-level key + :type tags: str + :param default: a default value for these tags + :type default: anything, optional + :return: None + :type: None + """ scope = self._scopes[key] tags = self._ensure_exclamation(tags) default = default if not default or default.startswith("!") else "!" + default @@ -93,7 +173,18 @@ def scope(self, key, *tags, default=None): scope.append(default) def load(self, content, validate=True): - """Parse yaml content.""" + """ + Parse yaml content. + + :param content: the content of a config file + :type content: str + :param validate: if set to ``False``, no validation will be performed. Tags can then occur anywhere. + :type validate: bool + :return: the parsed config + :type: dict + :raises lib50.InvalidConfigError: in case a tag is misplaced, or the content is not valid yaml. + :raises lib50.MissingToolError: in case the tool does not occur in the content. + """ # Try parsing the YAML with global tags try: config = yaml.load(content, Loader=self._loader(self._global_tags)) @@ -234,4 +325,5 @@ def _apply_scope(self, config, tags): @staticmethod def _ensure_exclamation(tags): + """Places an exclamation mark for each tag that does not already have one""" return [tag if tag.startswith("!") else "!" + tag for tag in tags] From 79cac9206f96b90cd339140e4c54cff6797be36b Mon Sep 17 00:00:00 2001 From: jelleas Date: Tue, 2 Jun 2020 18:39:33 +0200 Subject: [PATCH 26/64] lib50.crypto.verify docs --- lib50/crypto.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/lib50/crypto.py b/lib50/crypto.py index a1443b2..554fd22 100644 --- a/lib50/crypto.py +++ b/lib50/crypto.py @@ -17,15 +17,22 @@ def load_private_key(pem_str, password=None): def verify(payload, signature, public_key): """ - Verify payload using (base64 encoded) signature and verification key. verification_key should be obtained from load_public_key + Verify payload using (base64 encoded) signature and verification key. public_key should be obtained from load_public_key Uses RSA-PSS with SHA-512 and maximum salt length. The corresponding openssl command to create signatures that this function can verify is: - openssl dgst -sha512 -sigopt rsa_padding_mode:pss -sigopt rsa_pss_saltlen:-2 -sign | openssl base64 -A + :: - returns a boolean that is true iff the payload could be verified - """ + openssl dgst -sha512 -sigopt rsa_padding_mode:pss -sigopt rsa_pss_saltlen:-2 -sign | openssl base64 -A + :param payload: the payload + :type payload: str + :param signature: base64 encoded signature + :type signature: bytes + :param public_key: a public key from ``lib50.crypto.load_public_key`` + :return: True iff the payload could be verified + :type: bool + """ try: public_key.verify(signature=base64.b64decode(signature), data=payload, From 9bfe0d4f640ccb0c5e058a03287dd789fc60e951 Mon Sep 17 00:00:00 2001 From: jelleas Date: Wed, 3 Jun 2020 14:23:33 +0200 Subject: [PATCH 27/64] crypto docs --- lib50/crypto.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/lib50/crypto.py b/lib50/crypto.py index 554fd22..e0f7cae 100644 --- a/lib50/crypto.py +++ b/lib50/crypto.py @@ -1,3 +1,5 @@ +"""An API for verifying signed payloads such as check50 results.""" + import base64 from cryptography.exceptions import InvalidSignature @@ -8,10 +10,42 @@ from ._errors import InvalidSignatureError def load_public_key(pem_str): + """ + Load a public key from a PEM string. + + "PEM is an encapsulation format, meaning keys in it can actually be any of several different key types. + However these are all self-identifying, so you don’t need to worry about this detail. + PEM keys are recognizable because they all begin with + ``-----BEGIN {format}-----`` and end with ``-----END {format}-----``." + + - source: https://cryptography.io/en/latest/hazmat/primitives/asymmetric/serialization/#pem + + :param pem_str: the public key to load in PEM format + :type pem_str: str + :return: a key from ``cryptography.hazmat`` + :type: One of RSAPrivateKey, DSAPrivateKey, DHPrivateKey, or EllipticCurvePrivateKey + """ return serialization.load_pem_public_key(pem_str, backend=default_backend()) def load_private_key(pem_str, password=None): + """ + Load a private key from a PEM string. + + "PEM is an encapsulation format, meaning keys in it can actually be any of several different key types. + However these are all self-identifying, so you don’t need to worry about this detail. + PEM keys are recognizable because they all begin with + ``-----BEGIN {format}-----`` and end with ``-----END {format}-----``." + + - source: https://cryptography.io/en/latest/hazmat/primitives/asymmetric/serialization/#pem + + :param pem_str: the private key to load in PEM format + :type pem_str: str + :param password: a password to decode the pem_str + :type password: str, optional + :return: a key from ``cryptography.hazmat`` + :type: One of RSAPrivateKey, DSAPrivateKey, DHPrivateKey, or EllipticCurvePrivateKey + """ return serialization.load_pem_private_key(pem_str, password=password, backend=default_backend()) @@ -47,6 +81,15 @@ def verify(payload, signature, public_key): def sign(payload, private_key): + """ + Sign a payload with a private key. + + :param payload: the payload + :type payload: str + :param private_key: a private key from ``lib50.crypto.load_private_key`` + :return: base64 encoded signature + :type: bytes + """ return base64.b64encode( private_key.sign(data=payload, padding=padding.PSS( From 2c17d8e7e7cae134a81c774f3608f960f649c771 Mon Sep 17 00:00:00 2001 From: jelleas Date: Wed, 3 Jun 2020 14:48:51 +0200 Subject: [PATCH 28/64] writeup --- docs/source/api.rst | 2 +- docs/source/index.rst | 6 +++--- lib50/_api.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/source/api.rst b/docs/source/api.rst index 8c3bd80..fa06162 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -3,7 +3,7 @@ API docs ======== -.. _check50: +.. _lib50: lib50 ******* diff --git a/docs/source/index.rst b/docs/source/index.rst index de45443..1707dde 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -19,11 +19,11 @@ lib50 is CS50's library for common functionality shared between its tools. The library is, like most of CS50's projects, open-source, but its intention is to serve as an internal library for CS50's own tools. As such it is our current recommendation to not use lib50 as a dependency of one's own projects. -To promote reuse of functionality across CS50 tools, lib50 is designed to be tool agnostic. lib50 provides just the core functionality, but the semantics of that functionality are left up to the tool. For instance, submit50 adds the notion of a submission to a push to GitHub, whereas it is lib50 that provides the ``push`` function that ultimately handles the workflow with GitHub. Or per another example, lib50 provides the functionality to parse and validate ``.cs50.yml`` configuration files, but each individual tool (check50, submit50 and lab50) specifies their own options and handles their own logic. +To promote reuse of functionality across CS50 tools, lib50 is designed to be tool agnostic. lib50 provides just the core functionality, but the semantics of that functionality are left up to the tool. For instance, submit50 adds the notion of a submission to a push, whereas it is the ``lib50.push`` function that ultimately handles the workflow with git and GitHub. Or per another example, lib50 provides the functionality to parse and validate ``.cs50.yml`` configuration files, but each individual tool (check50, submit50 and lab50) specifies their own options and handles their own logic. -When looking for a piece of functionality that exists in other CS50 tools, odds are it lives in lib50. Increasingly so then, does the following apply: +With the overarching design goal to make it easy to add to or to change implementation choices, lib50 abstracts away from implementation details for other CS50 tools. Concepts such as slugs, git, `GitHub`, and ``.cs50.yml`` live only in lib50. Tools such as check50 interact only with lib50's API at a higher level of abstraction, such as ``lib50.push`` and ``lib50.config.Loader``. The idea being, that there is now a single point of change. For instance, one could add support for another host, such as GitLab perhaps, to ``lib50.push`` and all of CS50's tools could instantly make use of the new host. -"If it seems useful, it probably already exists in lib50" - Chad Sharp, 2019. +When looking for a piece of functionality that exists in other CS50 tools, odds are it lives in lib50. Installation diff --git a/lib50/_api.py b/lib50/_api.py index b41c8b9..54e905f 100644 --- a/lib50/_api.py +++ b/lib50/_api.py @@ -228,7 +228,7 @@ def files(patterns, exclude_tags=("exclude",), root="."): """ - Based on a list of patterns (lib50.config.TaggedValue) determine which files should be included and excluded. + Based on a list of patterns (``lib50.config.TaggedValue``) determine which files should be included and excluded. Any pattern tagged with a tag: * from ``include_tags`` will be included From 628acec899aadb5d5fbe364e8d6fa3958e6bdd66 Mon Sep 17 00:00:00 2001 From: jelleas Date: Wed, 3 Jun 2020 14:55:41 +0200 Subject: [PATCH 29/64] concepts --- docs/source/index.rst | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index 1707dde..7d15ed3 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -19,11 +19,21 @@ lib50 is CS50's library for common functionality shared between its tools. The library is, like most of CS50's projects, open-source, but its intention is to serve as an internal library for CS50's own tools. As such it is our current recommendation to not use lib50 as a dependency of one's own projects. -To promote reuse of functionality across CS50 tools, lib50 is designed to be tool agnostic. lib50 provides just the core functionality, but the semantics of that functionality are left up to the tool. For instance, submit50 adds the notion of a submission to a push, whereas it is the ``lib50.push`` function that ultimately handles the workflow with git and GitHub. Or per another example, lib50 provides the functionality to parse and validate ``.cs50.yml`` configuration files, but each individual tool (check50, submit50 and lab50) specifies their own options and handles their own logic. +The following concepts live primarily in lib50: + +* CS50 slugs +* git +* GitHub +* ``.cs50.yml`` config files +* signing and verification of payloads + -With the overarching design goal to make it easy to add to or to change implementation choices, lib50 abstracts away from implementation details for other CS50 tools. Concepts such as slugs, git, `GitHub`, and ``.cs50.yml`` live only in lib50. Tools such as check50 interact only with lib50's API at a higher level of abstraction, such as ``lib50.push`` and ``lib50.config.Loader``. The idea being, that there is now a single point of change. For instance, one could add support for another host, such as GitLab perhaps, to ``lib50.push`` and all of CS50's tools could instantly make use of the new host. +Design +****** + +To promote reuse of functionality across CS50 tools, lib50 is designed to be tool agnostic. lib50 provides just the core functionality, but the semantics of that functionality are left up to the tool. For instance, submit50 adds the notion of a submission to a push, whereas it is the ``lib50.push`` function that ultimately handles the workflow with git and GitHub. Or per another example, lib50 provides the functionality to parse and validate ``.cs50.yml`` configuration files, but each individual tool (check50, submit50 and lab50) specifies their own options and handles their own logic. -When looking for a piece of functionality that exists in other CS50 tools, odds are it lives in lib50. +With the overarching design goal to make it easy to add to or to change implementation choices, lib50 abstracts away from implementation details for other CS50 tools. Concepts such as slugs, git, GitHub, and ``.cs50.yml`` live only in lib50. Tools such as check50 interact only with lib50's API at a higher level of abstraction, such as ``lib50.push`` and ``lib50.config.Loader``. The idea being, that there is now a single point of change. For instance, one could add support for another host, such as GitLab perhaps, to ``lib50.push`` and all of CS50's tools could instantly make use of the new host. Installation From 38b55f1ae0c3ef10ac94de9ea146cee38b4bf100 Mon Sep 17 00:00:00 2001 From: jelleas Date: Wed, 3 Jun 2020 15:05:50 +0200 Subject: [PATCH 30/64] setup.py --- docs/Makefile | 2 +- setup.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/Makefile b/docs/Makefile index 445d1c2..41e5cda 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -3,7 +3,7 @@ # You can set these variables from the command line. SPHINXOPTS = -SPHINXBUILD = sphinx-build +SPHINXBUILD = sphinx-build SPHINXPROJ = lib50 SOURCEDIR = source BUILDDIR = build diff --git a/setup.py b/setup.py index d9de459..9a39ee7 100644 --- a/setup.py +++ b/setup.py @@ -18,6 +18,9 @@ license="GPLv3", description="This is lib50, CS50's own internal library used in many of its tools.", install_requires=["attrs>=18.1,<20", "pexpect>=4.6,<5", "pyyaml>=3.10,<6", "requests>=2.13,<3", "termcolor>=1.1,<2", "jellyfish>=0.7,<1", "cryptography>=2.7"], + extras_require = { + "develop": ["sphinx", "sphinx-autobuild", "sphinx_rtd_theme"] + }, keywords=["lib50"], name="lib50", python_requires=">= 3.6", From 6adbd7f170c5529cb8a4b00dd3d9eac8b2939b5c Mon Sep 17 00:00:00 2001 From: Jelle van Assema Date: Wed, 3 Jun 2020 22:07:33 +0200 Subject: [PATCH 31/64] insert path to project in conf.py for autodoc --- docs/source/conf.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 7127918..de86321 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -16,10 +16,10 @@ # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -# -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) + +import os +import sys +sys.path.insert(0, os.path.abspath('../../lib50')) # -- General configuration ------------------------------------------------ From 60b0e4796941be7565a0bdda0ee1aa538a79fa6c Mon Sep 17 00:00:00 2001 From: Jelle van Assema Date: Wed, 3 Jun 2020 22:17:09 +0200 Subject: [PATCH 32/64] add .readthedocs.yml --- .readthedocs.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .readthedocs.yml diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 0000000..06549f0 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,19 @@ +version: 2 + +sphinx: + builder: html + configuration: docs/conf.py + fail_on_warning: false + +formats: + - pdf + +python: + version: 3.6 + install: + - method: pip + path: . + extra_requirements: + - docs + - method: setuptools + path: package From b8179c209a907ff7bc44538dcea0e3e8c3d5aa7b Mon Sep 17 00:00:00 2001 From: Jelle van Assema Date: Wed, 3 Jun 2020 22:18:17 +0200 Subject: [PATCH 33/64] package => lib50 --- .readthedocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 06549f0..e8a5ec8 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -16,4 +16,4 @@ python: extra_requirements: - docs - method: setuptools - path: package + path: lib50 From c2e4d80d086c6b9f386ae14ab55e17356b62017d Mon Sep 17 00:00:00 2001 From: Jelle van Assema Date: Wed, 3 Jun 2020 22:19:35 +0200 Subject: [PATCH 34/64] lib50 => . --- .readthedocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index e8a5ec8..aef83d1 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -16,4 +16,4 @@ python: extra_requirements: - docs - method: setuptools - path: lib50 + path: . From 91a456af8f6fad9f626b502c19cf0bfcecc5a388 Mon Sep 17 00:00:00 2001 From: Jelle van Assema Date: Wed, 3 Jun 2020 22:21:54 +0200 Subject: [PATCH 35/64] docs => develop --- .readthedocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index aef83d1..82015ef 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -14,6 +14,6 @@ python: - method: pip path: . extra_requirements: - - docs + - develop - method: setuptools path: . From 40cdfe89e2917ec17401fea3309bef667918a19c Mon Sep 17 00:00:00 2001 From: Jelle van Assema Date: Wed, 3 Jun 2020 22:27:37 +0200 Subject: [PATCH 36/64] docs/source/conf.py --- .readthedocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 82015ef..3f1bce3 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -2,7 +2,7 @@ version: 2 sphinx: builder: html - configuration: docs/conf.py + configuration: docs/source/conf.py fail_on_warning: false formats: From 37282b2c33f14536f3d78a29f2307b6d731222ce Mon Sep 17 00:00:00 2001 From: "David J. Malan" Date: Wed, 3 Jun 2020 18:05:55 -0400 Subject: [PATCH 37/64] changed to dirhtml --- .readthedocs.yml | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 3f1bce3..8a7d83e 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,19 +1,16 @@ -version: 2 +python: + version: 3.7 + install: + - method: pip + path: . + extra_requirements: + - develop + - method: setuptools + path: . sphinx: - builder: html + builder: dirhtml configuration: docs/source/conf.py fail_on_warning: false -formats: - - pdf - -python: - version: 3.6 - install: - - method: pip - path: . - extra_requirements: - - develop - - method: setuptools - path: . +version: 2 From b9280faac3dfbad47adf1212f6895201bdaa536b Mon Sep 17 00:00:00 2001 From: jelleas Date: Thu, 4 Jun 2020 15:35:24 +0200 Subject: [PATCH 38/64] mimic cs50.readthedocs --- .readthedocs.yml | 7 +- docs/requirements.txt | 3 - docs/source/conf.py | 177 +++--------------------------------------- 3 files changed, 15 insertions(+), 172 deletions(-) delete mode 100644 docs/requirements.txt diff --git a/.readthedocs.yml b/.readthedocs.yml index 8a7d83e..2733687 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,3 +1,6 @@ +build: + image: latest + python: version: 3.7 install: @@ -5,12 +8,8 @@ python: path: . extra_requirements: - develop - - method: setuptools - path: . sphinx: builder: dirhtml - configuration: docs/source/conf.py - fail_on_warning: false version: 2 diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index 04c8577..0000000 --- a/docs/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -sphinx -sphinx-autobuild -sphinx_rtd_theme diff --git a/docs/source/conf.py b/docs/source/conf.py index de86321..b4b859f 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -1,175 +1,22 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# lib50 documentation build configuration file, created by -# sphinx-quickstart on Wed Jun 27 13:24:24 2018. -# -# This file is execfile()d with the current directory set to its -# containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. - import os import sys -sys.path.insert(0, os.path.abspath('../../lib50')) - +import time -# -- General configuration ------------------------------------------------ +_tool = "lib50" -# If your documentation needs a minimal Sphinx version, state it here. -# -# needs_sphinx = '1.0' +# Add path to module for autodoc +sys.path.insert(0, os.path.abspath(f'../../{_tool}')) -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. extensions = ['sphinx.ext.autodoc'] -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix(es) of source filenames. -# You can specify multiple suffix as a list of string: -# -# source_suffix = ['.rst', '.md'] -source_suffix = '.rst' - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -project = 'lib50' -copyright = '2018, CS50' -author = 'CS50' - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = '' -# The full version, including alpha/beta/rc tags. -release = '' - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# -# This is also used if you do content translation via gettext catalogs. -# Usually you set "language" from the command line for these cases. -language = None - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This patterns also effect to html_static_path and html_extra_path -exclude_patterns = [] - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' - -# If true, `todo` and `todoList` produce output, else they produce nothing. -todo_include_todos = False - - -# -- Options for HTML output ---------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# +html_css_files = ["https://cs50.readthedocs.io/_static/custom.css?" + str(round(time.time()))] +html_js_files = ["https://cs50.readthedocs.io/_static/custom.js?" + str(round(time.time()))] html_theme = "sphinx_rtd_theme" - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -# -html_theme_options = {"collapse_navigation" : False} - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = [] - -# Custom sidebar templates, must be a dictionary that maps document names -# to template names. -# -# This is required for the alabaster theme -# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars -html_sidebars = { - '**': [ - 'relations.html', # needs 'show_related': True theme option to display - 'searchbox.html', - ] +html_theme_options = { + "display_version": False, + "prev_next_buttons_location": False, + "sticky_navigation": False } +html_title = f'{_tool} Docs' - -# -- Options for HTMLHelp output ------------------------------------------ - -# Output file base name for HTML help builder. -htmlhelp_basename = 'lib50doc' - - -# -- Options for LaTeX output --------------------------------------------- - -latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - # - # 'papersize': 'letterpaper', - - # The font size ('10pt', '11pt' or '12pt'). - # - # 'pointsize': '10pt', - - # Additional stuff for the LaTeX preamble. - # - # 'preamble': '', - - # Latex figure (float) alignment - # - # 'figure_align': 'htbp', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - (master_doc, 'lib50.tex', 'lib50 Documentation', - 'CS50', 'manual'), -] - - -# -- Options for manual page output --------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'lib50', 'lib50 Documentation', - [author], 1) -] - - -# -- Options for Texinfo output ------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - (master_doc, 'lib50', 'lib50 Documentation', - author, 'lib50', 'One line description of project.', - 'Miscellaneous'), -] - -# Source: https://stackoverflow.com/questions/5599254/how-to-use-sphinxs-autodoc-to-document-a-classs-init-self-method -def skip(app, what, name, obj, would_skip, options): - if name == "__init__": - return False - return would_skip - -def setup(app): - app.connect("autodoc-skip-member", skip) +project = f'{_tool}' From ca41d702421250781663c3326f616be74e4e4981 Mon Sep 17 00:00:00 2001 From: jelleas Date: Thu, 4 Jun 2020 16:05:56 +0200 Subject: [PATCH 39/64] include __init__ in autodoc + list instance vars in Slug --- docs/source/conf.py | 10 ++++++++++ lib50/_api.py | 8 ++++++++ lib50/_errors.py | 7 ++++++- 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index b4b859f..1460e50 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -7,6 +7,16 @@ # Add path to module for autodoc sys.path.insert(0, os.path.abspath(f'../../{_tool}')) +# Include __init__ in autodoc +# Source: https://stackoverflow.com/questions/5599254/how-to-use-sphinxs-autodoc-to-document-a-classs-init-self-method +def skip(app, what, name, obj, would_skip, options): + if name == "__init__": + return False + return would_skip + +def setup(app): + app.connect("autodoc-skip-member", skip) + extensions = ['sphinx.ext.autodoc'] html_css_files = ["https://cs50.readthedocs.io/_static/custom.css?" + str(round(time.time()))] diff --git a/lib50/_api.py b/lib50/_api.py index 54e905f..d9dee7c 100644 --- a/lib50/_api.py +++ b/lib50/_api.py @@ -761,6 +761,14 @@ class Slug: ``lib50.Slug`` performs validation on the slug, by querrying GitHub, pulling in all branches, and then by finding a branch and problem that matches the slug. + :ivar str org: the GitHub organization + :ivar str repo: the GitHub repo + :ivar str branch: the branch in the repo + :ivar str problem: path to the problem, the directory containing ``.cs50.yml`` + :ivar str slug: string representation of the slug + :ivar bool offline: flag signalling whether the user is offline. If set to True, the slug is parsed locally. + :ivar str origin: GitHub url for org/repo including authentication. + Example usage:: from lib50._api import Slug diff --git a/lib50/_errors.py b/lib50/_errors.py index 9e454a8..2a95544 100644 --- a/lib50/_errors.py +++ b/lib50/_errors.py @@ -4,7 +4,12 @@ __all__ = ["Error", "InvalidSlugError", "MissingFilesError", "InvalidConfigError", "MissingToolError", "TimeoutError", "ConnectionError"] class Error(Exception): - """A generic lib50 Error. lib50 Errors can carry arbitrary data in the dict ``Error.payload``,""" + """ + A generic lib50 Error. + + :ivar dict payload: arbitrary data + + """ def __init__(self, *args, **kwargs): """""" super().__init__(*args, **kwargs) From 46f6e96d11532a669acd0aaa7547a22099be1afd Mon Sep 17 00:00:00 2001 From: Rongxin Liu Date: Mon, 13 Jul 2020 18:42:30 -0400 Subject: [PATCH 40/64] add --single-branch flag for git clone --- lib50/_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib50/_api.py b/lib50/_api.py index d9dee7c..c87c551 100644 --- a/lib50/_api.py +++ b/lib50/_api.py @@ -453,7 +453,7 @@ def prepare(tool, branch, user, included): git = Git().set(Git.working_area) # Clone just .git folder try: - _run(git.set(Git.cache)("clone --bare {repo} .git", repo=user.repo)) + _run(git.set(Git.cache)("clone --bare {repo} .git --single-branch", repo=user.repo)) except Error: msg = _("Make sure your username and/or password are valid and {} is enabled for your account. To enable {}, ").format(tool, tool) if user.org != DEFAULT_PUSH_ORG: From fd89934bf2418b10af268650efc426e5f9785472 Mon Sep 17 00:00:00 2001 From: Rongxin Liu Date: Thu, 16 Jul 2020 18:43:05 -0400 Subject: [PATCH 41/64] prioritize single branch cloning --- lib50/_api.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib50/_api.py b/lib50/_api.py index c87c551..3e30039 100644 --- a/lib50/_api.py +++ b/lib50/_api.py @@ -453,7 +453,10 @@ def prepare(tool, branch, user, included): git = Git().set(Git.working_area) # Clone just .git folder try: - _run(git.set(Git.cache)("clone --bare {repo} .git --single-branch", repo=user.repo)) + try: + _run(git.set(Git.cache)("clone --bare --single-branch --branch {branch} {repo} .git", repo=user.repo, branch=branch)) + except Error: + _run(git.set(Git.cache)("clone --bare {repo} .git", repo=user.repo)) except Error: msg = _("Make sure your username and/or password are valid and {} is enabled for your account. To enable {}, ").format(tool, tool) if user.org != DEFAULT_PUSH_ORG: From bcccfe622eb3ee80262312b59c7b98c32a61c667 Mon Sep 17 00:00:00 2001 From: Rongxin Liu Date: Fri, 17 Jul 2020 00:15:19 -0400 Subject: [PATCH 42/64] factor out git command --- lib50/_api.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib50/_api.py b/lib50/_api.py index 3e30039..b20e6bc 100644 --- a/lib50/_api.py +++ b/lib50/_api.py @@ -453,10 +453,11 @@ def prepare(tool, branch, user, included): git = Git().set(Git.working_area) # Clone just .git folder try: + git_command = f"clone --bare --single-branch {user.repo} .git" try: - _run(git.set(Git.cache)("clone --bare --single-branch --branch {branch} {repo} .git", repo=user.repo, branch=branch)) + _run(git.set(Git.cache)(f"{git_command} --branch {branch}")) except Error: - _run(git.set(Git.cache)("clone --bare {repo} .git", repo=user.repo)) + _run(git.set(Git.cache)(git_command)) except Error: msg = _("Make sure your username and/or password are valid and {} is enabled for your account. To enable {}, ").format(tool, tool) if user.org != DEFAULT_PUSH_ORG: From 6f23b4e95e8709ac63237f598312709effd8b476 Mon Sep 17 00:00:00 2001 From: Rongxin Liu Date: Fri, 17 Jul 2020 00:23:19 -0400 Subject: [PATCH 43/64] fetch with depth 1 for local() --- lib50/_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib50/_api.py b/lib50/_api.py index b20e6bc..a202006 100644 --- a/lib50/_api.py +++ b/lib50/_api.py @@ -141,7 +141,7 @@ def local(slug, offline=False, remove_origin=False, github_token=None): if not offline: # Get latest version of checks - _run(git("fetch origin {branch}", branch=slug.branch)) + _run(git("fetch origin {branch} --depth 1", branch=slug.branch)) # Tolerate checkout failure (e.g., when origin doesn't exist) From 50af1491dcc21517cdd8a85e24740f9c135f1676 Mon Sep 17 00:00:00 2001 From: jelleas Date: Fri, 17 Jul 2020 15:28:02 +0200 Subject: [PATCH 44/64] update local() docs --- lib50/_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib50/_api.py b/lib50/_api.py index a202006..ec997f6 100644 --- a/lib50/_api.py +++ b/lib50/_api.py @@ -108,6 +108,7 @@ def push(tool, slug, config_loader, repo=None, data=None, prompt=lambda question def local(slug, offline=False, remove_origin=False, github_token=None): """ Create/update local copy of the GitHub repo indentified by slug. + The local copy is shallow and single branch, it contains just the last commit on the branch identified by the slug. :param slug: the slug identifying a GitHub repo. :type slug: str @@ -143,7 +144,6 @@ def local(slug, offline=False, remove_origin=False, github_token=None): # Get latest version of checks _run(git("fetch origin {branch} --depth 1", branch=slug.branch)) - # Tolerate checkout failure (e.g., when origin doesn't exist) try: _run(git("checkout -f -B {branch} origin/{branch}", branch=slug.branch)) From 39a6718a9dd46b68453c6766dabe17ec02805bdd Mon Sep 17 00:00:00 2001 From: Kareem Zidane Date: Fri, 17 Jul 2020 12:58:11 -0400 Subject: [PATCH 45/64] renamed variable, moved arg, upped version --- lib50/_api.py | 8 ++++---- setup.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib50/_api.py b/lib50/_api.py index ec997f6..2c2b493 100644 --- a/lib50/_api.py +++ b/lib50/_api.py @@ -142,7 +142,7 @@ def local(slug, offline=False, remove_origin=False, github_token=None): if not offline: # Get latest version of checks - _run(git("fetch origin {branch} --depth 1", branch=slug.branch)) + _run(git("fetch origin --depth 1 {branch}", branch=slug.branch)) # Tolerate checkout failure (e.g., when origin doesn't exist) try: @@ -453,11 +453,11 @@ def prepare(tool, branch, user, included): git = Git().set(Git.working_area) # Clone just .git folder try: - git_command = f"clone --bare --single-branch {user.repo} .git" + clone_command = f"clone --bare --single-branch {user.repo} .git" try: - _run(git.set(Git.cache)(f"{git_command} --branch {branch}")) + _run(git.set(Git.cache)(f"{clone_command} --branch {branch}")) except Error: - _run(git.set(Git.cache)(git_command)) + _run(git.set(Git.cache)(clone_command)) except Error: msg = _("Make sure your username and/or password are valid and {} is enabled for your account. To enable {}, ").format(tool, tool) if user.org != DEFAULT_PUSH_ORG: diff --git a/setup.py b/setup.py index 9a39ee7..b69446f 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,6 @@ python_requires=">= 3.6", packages=["lib50"], url="https://github.com/cs50/lib50", - version="3.0.0", + version="3.0.1", include_package_data=True ) From 145862a856a57d5435212afaa3e9f9d394058b0a Mon Sep 17 00:00:00 2001 From: Chad Sharp Date: Sat, 20 Mar 2021 19:08:31 -0400 Subject: [PATCH 46/64] add files limit, prohibit absolute paths --- lib50/_api.py | 67 +++++++++++++++++++++++++++++----------------- lib50/_errors.py | 44 +++++++++++++++++++++++++----- tests/api_tests.py | 10 +++---- 3 files changed, 85 insertions(+), 36 deletions(-) diff --git a/lib50/_api.py b/lib50/_api.py index 2c2b493..b2b04b0 100644 --- a/lib50/_api.py +++ b/lib50/_api.py @@ -44,8 +44,10 @@ DEFAULT_PUSH_ORG = "me50" AUTH_URL = "https://submit.cs50.io" +DEFAULT_FILE_LIMIT = 100 -def push(tool, slug, config_loader, repo=None, data=None, prompt=lambda question, included, excluded: True): + +def push(tool, slug, config_loader, repo=None, data=None, prompt=lambda question, included, excluded: True, file_limit=DEFAULT_FILE_LIMIT): """ Pushes to Github in name of a tool. What should be pushed is configured by the tool and its configuration in the .cs50.yml file identified by the slug. @@ -90,7 +92,7 @@ def push(tool, slug, config_loader, repo=None, data=None, prompt=lambda question check_dependencies() # Connect to GitHub and parse the config files - remote, (honesty, included, excluded) = connect(slug, config_loader) + remote, (honesty, included, excluded) = connect(slug, config_loader, file_limit=file_limit) # Authenticate the user with GitHub, and prepare the submission with authenticate(remote["org"], repo=repo) as user, prepare(tool, slug, user, included): @@ -226,7 +228,8 @@ def files(patterns, require_tags=("require",), include_tags=("include",), exclude_tags=("exclude",), - root="."): + root=".", + limit=DEFAULT_FILE_LIMIT): """ Based on a list of patterns (``lib50.config.TaggedValue``) determine which files should be included and excluded. Any pattern tagged with a tag: @@ -245,6 +248,8 @@ def files(patterns, :type exclude_tags: list of strings, optional :param root: the root directory from which to look for files. Defaults to the current directory. :type root: str or pathlib.Path, optional + :param limit: Maximum number of files that can be globbed. + :type limit: int :return: all included files and all excluded files :type: tuple(set of strings, set of strings) @@ -275,7 +280,7 @@ def files(patterns, with cd(root): # Include everything but hidden paths by default - included = _glob("*") + included = _glob("*", limit=limit) excluded = set() if patterns: @@ -283,6 +288,10 @@ def files(patterns, # For each pattern for pattern in patterns: + if Path(pattern.value).is_absolute(): + raise Error(_("Cannot include/exclude absolute paths, but an absolute path ({}) was specified.") + .format(pattern.value)) + # Include all files that are tagged with !require if pattern.tag in require_tags: file = str(Path(pattern.value)) @@ -297,12 +306,12 @@ def files(patterns, included.add(file) # Include all files that are tagged with !include elif pattern.tag in include_tags: - new_included = _glob(pattern.value) + new_included = _glob(pattern.value, limit=limit) excluded -= new_included included.update(new_included) # Exclude all files that are tagged with !exclude elif pattern.tag in exclude_tags: - new_excluded = _glob(pattern.value) + new_excluded = _glob(pattern.value, limit=limit) included -= new_excluded excluded.update(new_excluded) @@ -322,7 +331,7 @@ def files(patterns, return included, excluded -def connect(slug, config_loader): +def connect(slug, config_loader, file_limit=DEFAULT_FILE_LIMIT): """ Connects to a GitHub repo indentified by slug. Then parses the ``.cs50.yml`` config file with the ``config_loader``. @@ -332,6 +341,8 @@ def connect(slug, config_loader): :type slug: str :param config_loader: a config loader that is able to parse the .cs50.yml config file for a tool. :type config_loader: lib50.config.Loader + :param file_limit: The maximum number of files that are allowed to be included. + :type file_limit: int :return: the remote configuration (org, message, callback, results), and the input for a prompt (honesty question, included files, excluded files) :type: tuple(dict, tuple(str, set, set)) :raises lib50.InvalidSlugError: if the slug is invalid for the tool @@ -373,13 +384,12 @@ def connect(slug, config_loader): honesty = config.get("honesty", True) # Figure out which files to include and exclude - included, excluded = files(config.get("files")) + included, excluded = files(config.get("files"), limit=file_limit) # Check that at least 1 file is staged if not included: raise Error(_("No files in this directory are expected by {}.".format(slug))) - return remote, (honesty, included, excluded) @@ -466,7 +476,7 @@ def prepare(tool, branch, user, included): msg += _("please go to {} in your web browser and try again.").format(AUTH_URL) msg += _((" If you're using GitHub two-factor authentication, you'll need to create and use a personal access token " - "with the \"repo\" scope instead of your password. See https://cs50.ly/github-2fa for more information!")) + "with the \"repo\" scope instead of your password. See https://cs50.ly/github-2fa for more information!")) raise Error(msg) @@ -712,7 +722,7 @@ class User: org = attr.ib() email = attr.ib(default=attr.Factory(lambda self: f"{self.name}@users.noreply.github.com", takes_self=True), - init=False) + init=False) class Git: @@ -784,6 +794,7 @@ class Slug: print(slug.problem) """ + def __init__(self, slug, offline=False, github_token=None): """Parse /// from slug.""" self.slug = self.normalize_case(slug) @@ -870,7 +881,6 @@ def normalize_case(slug): parts[1] = parts[1].lower() return "/".join(parts) - def __str__(self): return self.slug @@ -993,24 +1003,33 @@ def _run(command, quiet=False, timeout=None): return command_output -def _glob(pattern, skip_dirs=False): - """Glob pattern, expand directories, return all files that matched.""" +def _glob(pattern, skip_dirs=False, limit=DEFAULT_FILE_LIMIT): + """ + Glob pattern, expand directories, return iterator over matching files. + Throws ``lib50.TooManyFilesError`` if more than ``limit`` files are globbed. + """ # Implicit recursive iff no / in pattern and starts with * - if "/" not in pattern and pattern.startswith("*"): - files = glob.glob(f"**/{pattern}", recursive=True) - else: - files = glob.glob(pattern, recursive=True) + files = glob.iglob(f"**/{pattern}" if "/" not in pattern and pattern.startswith("*") + else pattern, recursive=True) - # Expand dirs all_files = set() + + def add_file(f): + fname = str(Path(f)) + all_files.add(fname) + if len(all_files) > limit: + raise TooManyFilesError(limit) + + # Expand dirs for file in files: if os.path.isdir(file) and not skip_dirs: - all_files.update(set(f for f in _glob(f"{file}/**/*", skip_dirs=True) if not os.path.isdir(f))) + for f in _glob(f"{file}/**/*", skip_dirs=True): + if not os.path.isdir(f): + add_file(f) else: - all_files.add(file) + add_file(file) - # Normalize all files - return {str(Path(f)) for f in all_files} + return all_files def _match_files(universe, pattern): @@ -1124,7 +1143,6 @@ def _authenticate_ssh(org, repo=None): except (pexpect.EOF, pexpect.TIMEOUT): return None - child.close() if i == 0: @@ -1159,7 +1177,6 @@ def _authenticate_https(org, repo=None): child.close() child.exitstatus = 0 - if password is None: username = _prompt_username(_("GitHub username: ")) password = _prompt_password(_("GitHub password: ")) diff --git a/lib50/_errors.py b/lib50/_errors.py index 2a95544..531e79c 100644 --- a/lib50/_errors.py +++ b/lib50/_errors.py @@ -1,7 +1,9 @@ import os from . import _ -__all__ = ["Error", "InvalidSlugError", "MissingFilesError", "InvalidConfigError", "MissingToolError", "TimeoutError", "ConnectionError"] +__all__ = ["Error", "InvalidSlugError", "MissingFilesError", "TooManyFilesError", + "InvalidConfigError", "MissingToolError", "TimeoutError", "ConnectionError"] + class Error(Exception): """ @@ -10,52 +12,82 @@ class Error(Exception): :ivar dict payload: arbitrary data """ + def __init__(self, *args, **kwargs): """""" super().__init__(*args, **kwargs) self.payload = {} + class InvalidSlugError(Error): """A ``lib50.Error`` signalling that a slug is invalid.""" pass + class MissingFilesError(Error): """ - A ``ib50.Error`` signalling that files are missing. + A ``lib50.Error`` signaling that files are missing. This error's payload has a ``files`` and ``dir`` key. ``MissingFilesError.payload["files"]`` are all the missing files in a list of strings. ``MissingFilesError.payload["dir"]`` is the current working directory (cwd) from when this error was raised. """ - def __init__(self, files): + + def __init__(self, files, dir=None): """ :param files: the missing files that caused the error :type files: list of string(s) or Pathlib.path(s) """ - cwd = os.getcwd().replace(os.path.expanduser("~"), "~", 1) + if dir is None: + dir = os.path.expanduser(os.getcwd()) + super().__init__("{}\n{}\n{}".format( _("You seem to be missing these required files:"), "\n".join(files), - _("You are currently in: {}, did you perhaps intend another directory?".format(cwd)) + _("You are currently in: {}, did you perhaps intend another directory?".format(dir)) + )) + self.payload.update(files=files, dir=dir) + + +class TooManyFilesError(Error): + """ + A ``lib50.Error`` signaling that too many files were attempted to be included. + The error's payload has a ``dir`` and a ``limit`` key. + ``TooManyFilesError.payload["dir"]`` is the directory in which the attempted submission occured. + ``TooManyFilesError.payload["limit"]`` is the max number of files allowed + """ + + def __init__(self, limit, dir=None): + + if dir is None: + dir = os.path.expanduser(os.getcwd()) + + super().__init__("{}\n{}".format( + _("Looks like you are attempting to include too many (> {}) files.").format(limit), + _("You are currently in: {}, did you perhaps intend another directory?".format(dir)) )) - self.payload.update(files=files, dir=cwd) + self.payload.update(limit=limit, dir=dir) class InvalidConfigError(Error): """A ``lib50.Error`` signalling that a config is invalid.""" pass + class MissingToolError(InvalidConfigError): """A more specific ``lib50.InvalidConfigError`` signalling that an entry for a tool is missing in the config.""" pass + class TimeoutError(Error): """A ``lib50.Error`` signalling a timeout has occured.""" pass + class ConnectionError(Error): """A ``lib50.Error`` signalling a connection has errored.""" pass + class InvalidSignatureError(Error): """A ``lib50.Error`` signalling the signature of a payload is invalid.""" pass diff --git a/tests/api_tests.py b/tests/api_tests.py index 94e2cba..1eeb6a3 100644 --- a/tests/api_tests.py +++ b/tests/api_tests.py @@ -110,7 +110,7 @@ def test_offline(self): os.makedirs(path) os.chdir(pathlib.Path(lib50.get_local_path()) / "foo" / "bar") - subprocess.check_output(["git", "init"]) + subprocess.check_output(["git", "init", "-b", "main"]) os.chdir(path) @@ -119,11 +119,11 @@ def test_offline(self): subprocess.check_output(["git", "add", ".cs50.yaml"]) out = subprocess.check_output(["git", "commit", "-m", "\"qux\""]) - slug = lib50._api.Slug("foo/bar/master/baz", offline=True) - self.assertEqual(slug.slug, "foo/bar/master/baz") + slug = lib50._api.Slug("foo/bar/main/baz", offline=True) + self.assertEqual(slug.slug, "foo/bar/main/baz") self.assertEqual(slug.org, "foo") self.assertEqual(slug.repo, "bar") - self.assertEqual(slug.branch, "master") + self.assertEqual(slug.branch, "main") self.assertEqual(slug.problem, pathlib.Path("baz")) finally: lib50.set_local_path(old_local_path) @@ -256,7 +256,7 @@ def tearDown(self): def test_one_local_slug(self): slugs = list(lib50.get_local_slugs("foo50")) self.assertEqual(len(slugs), 1) - self.assertEqual(slugs[0], "foo/bar/master/baz") + self.assertEqual(slugs[0], "foo/bar/main/baz") if __name__ == '__main__': From da1bae33c05d0c19820a9cad2b5d52834903ba6e Mon Sep 17 00:00:00 2001 From: Chad Sharp Date: Sat, 20 Mar 2021 19:12:06 -0400 Subject: [PATCH 47/64] fix tests --- tests/api_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/api_tests.py b/tests/api_tests.py index 1eeb6a3..8f4d6ce 100644 --- a/tests/api_tests.py +++ b/tests/api_tests.py @@ -243,7 +243,7 @@ def setUp(self): os.makedirs(path) with open(path / ".cs50.yml", "w") as f: f.write("foo50: true\n") - pexpect.run(f"git -C {path.parent.parent} init") + pexpect.run(f"git -C {path.parent.parent} init -b main") pexpect.run(f"git -C {path.parent.parent} add .") pexpect.run(f"git -C {path.parent.parent} commit -m \"message\"") self.debug_output = [] From 764988d249aaf867fbd370e0e67c660b3c265563 Mon Sep 17 00:00:00 2001 From: Jelleas Date: Mon, 22 Mar 2021 13:14:37 +0100 Subject: [PATCH 48/64] older version of git dont support git init -b --- tests/api_tests.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/api_tests.py b/tests/api_tests.py index 8f4d6ce..9e96b02 100644 --- a/tests/api_tests.py +++ b/tests/api_tests.py @@ -83,7 +83,7 @@ def test_wrong_format(self): def test_case(self): with self.assertRaises(lib50._api.InvalidSlugError): lib50._api.Slug("cs50/lib50/TESTS/bar") - self.assertEquals(lib50._api.Slug("CS50/LiB50/tests/bar").slug, "cs50/lib50/tests/bar") + self.assertEqual(lib50._api.Slug("CS50/LiB50/tests/bar").slug, "cs50/lib50/tests/bar") def test_online(self): if os.environ.get("TRAVIS") == "true": @@ -110,7 +110,8 @@ def test_offline(self): os.makedirs(path) os.chdir(pathlib.Path(lib50.get_local_path()) / "foo" / "bar") - subprocess.check_output(["git", "init", "-b", "main"]) + subprocess.check_output(["git", "init"]) + subprocess.check_output(["git", "checkout", "-b", "main"]) os.chdir(path) @@ -243,7 +244,8 @@ def setUp(self): os.makedirs(path) with open(path / ".cs50.yml", "w") as f: f.write("foo50: true\n") - pexpect.run(f"git -C {path.parent.parent} init -b main") + pexpect.run(f"git -C {path.parent.parent} init") + pexpect.run(f"git -C {path.parent.parent} checkout -b main") pexpect.run(f"git -C {path.parent.parent} add .") pexpect.run(f"git -C {path.parent.parent} commit -m \"message\"") self.debug_output = [] From fa50c53b14d8f8264d798b37b4f5b31710036aa4 Mon Sep 17 00:00:00 2001 From: Jelleas Date: Mon, 22 Mar 2021 13:49:56 +0100 Subject: [PATCH 49/64] file_limit description, raise limit to 10000, rephrase error --- lib50/_api.py | 5 ++++- lib50/_errors.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lib50/_api.py b/lib50/_api.py index b2b04b0..321981a 100644 --- a/lib50/_api.py +++ b/lib50/_api.py @@ -44,7 +44,8 @@ DEFAULT_PUSH_ORG = "me50" AUTH_URL = "https://submit.cs50.io" -DEFAULT_FILE_LIMIT = 100 + +DEFAULT_FILE_LIMIT = 10000 def push(tool, slug, config_loader, repo=None, data=None, prompt=lambda question, included, excluded: True, file_limit=DEFAULT_FILE_LIMIT): @@ -67,6 +68,8 @@ def push(tool, slug, config_loader, repo=None, data=None, prompt=lambda question :type data: dict of strings, optional :param prompt: a prompt shown just before the push. In case this prompt returns false, the push is aborted. This lambda function has access to an honesty prompt configured in .cs50,yml, and all files that will be included and excluded in the push. :type prompt: lambda str, list, list => bool, optional + :param file_limit: maximum number of files to be matched by any globbing pattern. + :type file_limit: int :return: GitHub username and the commit hash :type: tuple(str, str) diff --git a/lib50/_errors.py b/lib50/_errors.py index 531e79c..5f77f11 100644 --- a/lib50/_errors.py +++ b/lib50/_errors.py @@ -62,7 +62,7 @@ def __init__(self, limit, dir=None): dir = os.path.expanduser(os.getcwd()) super().__init__("{}\n{}".format( - _("Looks like you are attempting to include too many (> {}) files.").format(limit), + _("Looks like you are in a directory with too many (> {}) files.").format(limit), _("You are currently in: {}, did you perhaps intend another directory?".format(dir)) )) self.payload.update(limit=limit, dir=dir) From b2686c641c52d6b658feeb960f32f12c76ef96ca Mon Sep 17 00:00:00 2001 From: Jelleas Date: Mon, 22 Mar 2021 14:02:39 +0100 Subject: [PATCH 50/64] rename file_limit glob_limit --- lib50/_api.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/lib50/_api.py b/lib50/_api.py index 321981a..db30ccb 100644 --- a/lib50/_api.py +++ b/lib50/_api.py @@ -45,10 +45,10 @@ AUTH_URL = "https://submit.cs50.io" -DEFAULT_FILE_LIMIT = 10000 +DEFAULT_GLOB_LIMIT = 10000 -def push(tool, slug, config_loader, repo=None, data=None, prompt=lambda question, included, excluded: True, file_limit=DEFAULT_FILE_LIMIT): +def push(tool, slug, config_loader, repo=None, data=None, prompt=lambda question, included, excluded: True, glob_limit=DEFAULT_GLOB_LIMIT): """ Pushes to Github in name of a tool. What should be pushed is configured by the tool and its configuration in the .cs50.yml file identified by the slug. @@ -68,8 +68,8 @@ def push(tool, slug, config_loader, repo=None, data=None, prompt=lambda question :type data: dict of strings, optional :param prompt: a prompt shown just before the push. In case this prompt returns false, the push is aborted. This lambda function has access to an honesty prompt configured in .cs50,yml, and all files that will be included and excluded in the push. :type prompt: lambda str, list, list => bool, optional - :param file_limit: maximum number of files to be matched by any globbing pattern. - :type file_limit: int + :param glob_limit: maximum number of files to be matched by any globbing pattern. + :type glob_limit: int :return: GitHub username and the commit hash :type: tuple(str, str) @@ -95,7 +95,7 @@ def push(tool, slug, config_loader, repo=None, data=None, prompt=lambda question check_dependencies() # Connect to GitHub and parse the config files - remote, (honesty, included, excluded) = connect(slug, config_loader, file_limit=file_limit) + remote, (honesty, included, excluded) = connect(slug, config_loader, glob_limit=DEFAULT_GLOB_LIMIT) # Authenticate the user with GitHub, and prepare the submission with authenticate(remote["org"], repo=repo) as user, prepare(tool, slug, user, included): @@ -232,7 +232,7 @@ def files(patterns, include_tags=("include",), exclude_tags=("exclude",), root=".", - limit=DEFAULT_FILE_LIMIT): + limit=DEFAULT_GLOB_LIMIT): """ Based on a list of patterns (``lib50.config.TaggedValue``) determine which files should be included and excluded. Any pattern tagged with a tag: @@ -334,7 +334,7 @@ def files(patterns, return included, excluded -def connect(slug, config_loader, file_limit=DEFAULT_FILE_LIMIT): +def connect(slug, config_loader, glob_limit=DEFAULT_GLOB_LIMIT): """ Connects to a GitHub repo indentified by slug. Then parses the ``.cs50.yml`` config file with the ``config_loader``. @@ -344,8 +344,8 @@ def connect(slug, config_loader, file_limit=DEFAULT_FILE_LIMIT): :type slug: str :param config_loader: a config loader that is able to parse the .cs50.yml config file for a tool. :type config_loader: lib50.config.Loader - :param file_limit: The maximum number of files that are allowed to be included. - :type file_limit: int + :param glob_limit: The maximum number of files that are allowed to be included. + :type glob_limit: int :return: the remote configuration (org, message, callback, results), and the input for a prompt (honesty question, included files, excluded files) :type: tuple(dict, tuple(str, set, set)) :raises lib50.InvalidSlugError: if the slug is invalid for the tool @@ -387,7 +387,7 @@ def connect(slug, config_loader, file_limit=DEFAULT_FILE_LIMIT): honesty = config.get("honesty", True) # Figure out which files to include and exclude - included, excluded = files(config.get("files"), limit=file_limit) + included, excluded = files(config.get("files"), limit=glob_limit) # Check that at least 1 file is staged if not included: @@ -1006,7 +1006,7 @@ def _run(command, quiet=False, timeout=None): return command_output -def _glob(pattern, skip_dirs=False, limit=DEFAULT_FILE_LIMIT): +def _glob(pattern, skip_dirs=False, limit=DEFAULT_GLOB_LIMIT): """ Glob pattern, expand directories, return iterator over matching files. Throws ``lib50.TooManyFilesError`` if more than ``limit`` files are globbed. From b5908526b413f6f16823cffde1f9e3dd031ecb01 Mon Sep 17 00:00:00 2001 From: Chad Sharp Date: Mon, 22 Mar 2021 16:45:02 -0400 Subject: [PATCH 51/64] improve absolute path checking --- lib50/_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib50/_api.py b/lib50/_api.py index db30ccb..5935bf7 100644 --- a/lib50/_api.py +++ b/lib50/_api.py @@ -291,8 +291,8 @@ def files(patterns, # For each pattern for pattern in patterns: - if Path(pattern.value).is_absolute(): - raise Error(_("Cannot include/exclude absolute paths, but an absolute path ({}) was specified.") + if Path(pattern.value).expanduser().resolve().is_relative_to(Path.cwd()): + raise Error(_("Cannot include/exclude paths outside the current directory, but such a path ({}) was specified.") .format(pattern.value)) # Include all files that are tagged with !require From 6c6e551f2533bd0cf79a05118094b27cff0a9e2c Mon Sep 17 00:00:00 2001 From: Chad Sharp Date: Mon, 22 Mar 2021 16:48:43 -0400 Subject: [PATCH 52/64] missing not --- lib50/_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib50/_api.py b/lib50/_api.py index 5935bf7..cfde1a6 100644 --- a/lib50/_api.py +++ b/lib50/_api.py @@ -291,7 +291,7 @@ def files(patterns, # For each pattern for pattern in patterns: - if Path(pattern.value).expanduser().resolve().is_relative_to(Path.cwd()): + if not Path(pattern.value).expanduser().resolve().is_relative_to(Path.cwd()): raise Error(_("Cannot include/exclude paths outside the current directory, but such a path ({}) was specified.") .format(pattern.value)) From abe5f0938df5784efd5786855bc4cfe9df73b4cb Mon Sep 17 00:00:00 2001 From: Chad Sharp Date: Mon, 22 Mar 2021 16:55:40 -0400 Subject: [PATCH 53/64] is_relative_to is 3.9+ only --- lib50/_api.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/lib50/_api.py b/lib50/_api.py index cfde1a6..9889595 100644 --- a/lib50/_api.py +++ b/lib50/_api.py @@ -291,7 +291,7 @@ def files(patterns, # For each pattern for pattern in patterns: - if not Path(pattern.value).expanduser().resolve().is_relative_to(Path.cwd()): + if not _is_relative_to(Path(pattern.value).expanduser().resolve(), Path.cwd()): raise Error(_("Cannot include/exclude paths outside the current directory, but such a path ({}) was specified.") .format(pattern.value)) @@ -1260,6 +1260,15 @@ def _prompt_password(prompt="Password: "): return password_string +def _is_relative_to(path, *others): + """The is_relative_to method for Paths is Python 3.9+ so we implement it here.""" + try: + path.relative_to(*others) + return True + except ValueError: + return False + + @contextlib.contextmanager def _no_echo_stdin(): """ From 1a7f3f263a4abc8a0879300c38aa8830ab9d83a5 Mon Sep 17 00:00:00 2001 From: Jelleas Date: Mon, 22 Mar 2021 22:32:15 +0100 Subject: [PATCH 54/64] mv back to file_limit --- lib50/_api.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/lib50/_api.py b/lib50/_api.py index 9889595..18bc088 100644 --- a/lib50/_api.py +++ b/lib50/_api.py @@ -45,10 +45,10 @@ AUTH_URL = "https://submit.cs50.io" -DEFAULT_GLOB_LIMIT = 10000 +DEFAULT_FILE_LIMIT = 10000 -def push(tool, slug, config_loader, repo=None, data=None, prompt=lambda question, included, excluded: True, glob_limit=DEFAULT_GLOB_LIMIT): +def push(tool, slug, config_loader, repo=None, data=None, prompt=lambda question, included, excluded: True, file_limit=DEFAULT_FILE_LIMIT): """ Pushes to Github in name of a tool. What should be pushed is configured by the tool and its configuration in the .cs50.yml file identified by the slug. @@ -68,8 +68,8 @@ def push(tool, slug, config_loader, repo=None, data=None, prompt=lambda question :type data: dict of strings, optional :param prompt: a prompt shown just before the push. In case this prompt returns false, the push is aborted. This lambda function has access to an honesty prompt configured in .cs50,yml, and all files that will be included and excluded in the push. :type prompt: lambda str, list, list => bool, optional - :param glob_limit: maximum number of files to be matched by any globbing pattern. - :type glob_limit: int + :param file_limit: maximum number of files to be matched by any globbing pattern. + :type file_limit: int :return: GitHub username and the commit hash :type: tuple(str, str) @@ -95,7 +95,7 @@ def push(tool, slug, config_loader, repo=None, data=None, prompt=lambda question check_dependencies() # Connect to GitHub and parse the config files - remote, (honesty, included, excluded) = connect(slug, config_loader, glob_limit=DEFAULT_GLOB_LIMIT) + remote, (honesty, included, excluded) = connect(slug, config_loader, file_limit=DEFAULT_FILE_LIMIT) # Authenticate the user with GitHub, and prepare the submission with authenticate(remote["org"], repo=repo) as user, prepare(tool, slug, user, included): @@ -232,7 +232,7 @@ def files(patterns, include_tags=("include",), exclude_tags=("exclude",), root=".", - limit=DEFAULT_GLOB_LIMIT): + limit=DEFAULT_FILE_LIMIT): """ Based on a list of patterns (``lib50.config.TaggedValue``) determine which files should be included and excluded. Any pattern tagged with a tag: @@ -334,7 +334,7 @@ def files(patterns, return included, excluded -def connect(slug, config_loader, glob_limit=DEFAULT_GLOB_LIMIT): +def connect(slug, config_loader, file_limit=DEFAULT_FILE_LIMIT): """ Connects to a GitHub repo indentified by slug. Then parses the ``.cs50.yml`` config file with the ``config_loader``. @@ -344,8 +344,8 @@ def connect(slug, config_loader, glob_limit=DEFAULT_GLOB_LIMIT): :type slug: str :param config_loader: a config loader that is able to parse the .cs50.yml config file for a tool. :type config_loader: lib50.config.Loader - :param glob_limit: The maximum number of files that are allowed to be included. - :type glob_limit: int + :param file_limit: The maximum number of files that are allowed to be included. + :type file_limit: int :return: the remote configuration (org, message, callback, results), and the input for a prompt (honesty question, included files, excluded files) :type: tuple(dict, tuple(str, set, set)) :raises lib50.InvalidSlugError: if the slug is invalid for the tool @@ -387,7 +387,7 @@ def connect(slug, config_loader, glob_limit=DEFAULT_GLOB_LIMIT): honesty = config.get("honesty", True) # Figure out which files to include and exclude - included, excluded = files(config.get("files"), limit=glob_limit) + included, excluded = files(config.get("files"), limit=file_limit) # Check that at least 1 file is staged if not included: @@ -1006,7 +1006,7 @@ def _run(command, quiet=False, timeout=None): return command_output -def _glob(pattern, skip_dirs=False, limit=DEFAULT_GLOB_LIMIT): +def _glob(pattern, skip_dirs=False, limit=DEFAULT_FILE_LIMIT): """ Glob pattern, expand directories, return iterator over matching files. Throws ``lib50.TooManyFilesError`` if more than ``limit`` files are globbed. From 89e8b4d99e822c49e576756769c7abc1b42ee274 Mon Sep 17 00:00:00 2001 From: Jelleas Date: Mon, 2 Aug 2021 18:12:01 +0200 Subject: [PATCH 55/64] initial SSH passphrases support --- lib50/_api.py | 212 +-------------------------------- lib50/authenticate.py | 267 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 272 insertions(+), 207 deletions(-) create mode 100644 lib50/authenticate.py diff --git a/lib50/_api.py b/lib50/_api.py index 18bc088..3dccdaf 100644 --- a/lib50/_api.py +++ b/lib50/_api.py @@ -1,11 +1,6 @@ -import collections import contextlib -import copy -import datetime import fnmatch -import gettext import glob -import itertools import logging import os from pathlib import Path @@ -17,9 +12,6 @@ import sys import tempfile import threading -import termios -import time -import tty import functools import attr @@ -27,10 +19,10 @@ import pexpect import requests import termcolor -import yaml from . import _, get_local_path from ._errors import * +from .authenticate import authenticate, logout, _run_authenticated from . import config as lib50_config __all__ = ["push", "local", "working_area", "files", "connect", @@ -40,7 +32,6 @@ logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) -_CREDENTIAL_SOCKET = Path("~/.git-credential-cache/lib50").expanduser() DEFAULT_PUSH_ORG = "me50" AUTH_URL = "https://submit.cs50.io" @@ -396,37 +387,6 @@ def connect(slug, config_loader, file_limit=DEFAULT_FILE_LIMIT): return remote, (honesty, included, excluded) -@contextlib.contextmanager -def authenticate(org, repo=None): - """ - A contextmanager that authenticates a user with GitHub via SSH if possible, otherwise via HTTPS. - - :param org: GitHub organisation to authenticate with - :type org: str - :param repo: GitHub repo (part of the org) to authenticate with. Default is the user's GitHub login. - :type repo: str, optional - :return: an authenticated user - :type: lib50.User - - Example usage:: - - from lib50 import authenticate - - with authenticate("me50") as user: - print(user.name) - - """ - with ProgressBar(_("Authenticating")) as progress_bar: - user = _authenticate_ssh(org, repo=repo) - progress_bar.stop() - if user is None: - # SSH auth failed, fallback to HTTPS - with _authenticate_https(org, repo=repo) as user: - yield user - else: - yield user - - @contextlib.contextmanager def prepare(tool, branch, user, included): """ @@ -464,13 +424,14 @@ def prepare(tool, branch, user, included): with ProgressBar(_("Verifying")): Git.working_area = f"-C {shlex.quote(str(area))}" git = Git().set(Git.working_area) + # Clone just .git folder try: clone_command = f"clone --bare --single-branch {user.repo} .git" try: - _run(git.set(Git.cache)(f"{clone_command} --branch {branch}")) + _run_authenticated(user, git.set(Git.cache)(f"{clone_command} --branch {branch}")) except Error: - _run(git.set(Git.cache)(clone_command)) + _run_authenticated(user, git.set(Git.cache)(clone_command)) except Error: msg = _("Make sure your username and/or password are valid and {} is enabled for your account. To enable {}, ").format(tool, tool) if user.org != DEFAULT_PUSH_ORG: @@ -539,7 +500,6 @@ def upload(branch, user, tool, data): upload(branch, user, tool, {tool:True}) """ - with ProgressBar(_("Uploading")): commit_message = _("automated commit by {}").format(tool) @@ -550,7 +510,7 @@ def upload(branch, user, tool, data): # Commit + push git = Git().set(Git.working_area) _run(git("commit -m {msg} --allow-empty", msg=commit_message)) - _run(git.set(Git.cache)("push origin {branch}", branch=branch)) + _run_authenticated(user, git.set(Git.cache)("push origin {branch}", branch=branch)) commit_hash = _run(git("rev-parse HEAD")) return user.name, commit_hash @@ -707,27 +667,6 @@ def check_dependencies(): raise Error(_("You have an old version of git. Install version 2.7 or later, then re-run!")) -def logout(): - """ - Log out from git. - - :return: None - :type: None - """ - _run(f"git credential-cache --socket {_CREDENTIAL_SOCKET} exit") - - -@attr.s(slots=True) -class User: - """An authenticated GitHub user that has write access to org/repo.""" - name = attr.ib() - repo = attr.ib() - org = attr.ib() - email = attr.ib(default=attr.Factory(lambda self: f"{self.name}@users.noreply.github.com", - takes_self=True), - init=False) - - class Git: """ A stateful helper class for formatting git commands. @@ -1133,133 +1072,6 @@ def _lfs_add(files, git): _run(git("add --force .gitattributes")) -def _authenticate_ssh(org, repo=None): - """Try authenticating via ssh, if succesful yields a User, otherwise raises Error.""" - # Require ssh-agent - child = pexpect.spawn("ssh -p443 -T git@ssh.github.com", encoding="utf8") - # GitHub prints 'Hi {username}!...' when attempting to get shell access - try: - i = child.expect(["Hi (.+)! You've successfully authenticated", - "Enter passphrase for key", - "Permission denied", - "Are you sure you want to continue connecting"]) - except (pexpect.EOF, pexpect.TIMEOUT): - return None - - child.close() - - if i == 0: - username = child.match.groups()[0] - else: - return None - - return User(name=username, - repo=f"ssh://git@ssh.github.com:443/{org}/{username if repo is None else repo}", - org=org) - - -@contextlib.contextmanager -def _authenticate_https(org, repo=None): - """Try authenticating via HTTPS, if succesful yields User, otherwise raises Error.""" - _CREDENTIAL_SOCKET.parent.mkdir(mode=0o700, exist_ok=True) - try: - Git.cache = f"-c credential.helper= -c credential.helper='cache --socket {_CREDENTIAL_SOCKET}'" - git = Git().set(Git.cache) - - # Get credentials from cache if possible - with _spawn(git("credential fill"), quiet=True) as child: - child.sendline("protocol=https") - child.sendline("host=github.com") - child.sendline("") - i = child.expect(["Username for '.+'", "Password for '.+'", - "username=([^\r]+)\r\npassword=([^\r]+)\r\n"]) - if i == 2: - username, password = child.match.groups() - else: - username = password = None - child.close() - child.exitstatus = 0 - - if password is None: - username = _prompt_username(_("GitHub username: ")) - password = _prompt_password(_("GitHub password: ")) - - # Credentials are correct, best cache them - with _spawn(git("-c credentialcache.ignoresighup=true credential approve"), quiet=True) as child: - child.sendline("protocol=https") - child.sendline("host=github.com") - child.sendline(f"path={org}/{username}") - child.sendline(f"username={username}") - child.sendline(f"password={password}") - child.sendline("") - - yield User(name=username, - repo=f"https://{username}@github.com/{org}/{username if repo is None else repo}", - org=org) - except BaseException: - # Some error occured while this context manager is active, best forget credentials. - logout() - raise - - -def _prompt_username(prompt="Username: "): - """Prompt the user for username.""" - try: - while True: - username = input(prompt).strip() - if not username: - print("Username cannot be empty, please try again.") - elif "@" in username: - print("Please enter your GitHub username, not email.") - else: - return username - except EOFError: - print() - - -def _prompt_password(prompt="Password: "): - """Prompt the user for password, printing asterisks for each character""" - print(prompt, end="", flush=True) - password_bytes = [] - password_string = "" - - with _no_echo_stdin(): - while True: - # Read one byte - ch = sys.stdin.buffer.read(1)[0] - # If user presses Enter or ctrl-d - if ch in (ord("\r"), ord("\n"), 4): - print("\r") - break - # Del - elif ch == 127: - if len(password_string) > 0: - print("\b \b", end="", flush=True) - # Remove last char and its corresponding bytes - password_string = password_string[:-1] - password_bytes = list(password_string.encode("utf8")) - # Ctrl-c - elif ch == 3: - print("^C", end="", flush=True) - raise KeyboardInterrupt - else: - password_bytes.append(ch) - - # If byte added concludes a utf8 char, print * - try: - password_string = bytes(password_bytes).decode("utf8") - except UnicodeDecodeError: - pass - else: - print("*", end="", flush=True) - - if not password_string: - print("Password cannot be empty, please try again.") - return _prompt_password(prompt) - - return password_string - - def _is_relative_to(path, *others): """The is_relative_to method for Paths is Python 3.9+ so we implement it here.""" try: @@ -1268,17 +1080,3 @@ def _is_relative_to(path, *others): except ValueError: return False - -@contextlib.contextmanager -def _no_echo_stdin(): - """ - On Unix only, have stdin not echo input. - https://stackoverflow.com/questions/510357/python-read-a-single-character-from-the-user - """ - fd = sys.stdin.fileno() - old_settings = termios.tcgetattr(fd) - tty.setraw(fd) - try: - yield - finally: - termios.tcsetattr(fd, termios.TCSADRAIN, old_settings) diff --git a/lib50/authenticate.py b/lib50/authenticate.py new file mode 100644 index 0000000..ab2f376 --- /dev/null +++ b/lib50/authenticate.py @@ -0,0 +1,267 @@ +import contextlib +import enum +import pexpect +import sys +import termios +import tty + +from pathlib import Path + +import attr + +from . import _ +from . import _api as api + + +_CREDENTIAL_SOCKET = Path("~/.git-credential-cache/lib50").expanduser() + + +@attr.s(slots=True) +class User: + """An authenticated GitHub user that has write access to org/repo.""" + name = attr.ib() + repo = attr.ib() + org = attr.ib() + passphrase = attr.ib() + email = attr.ib(default=attr.Factory(lambda self: f"{self.name}@users.noreply.github.com", + takes_self=True), + init=False) + +@contextlib.contextmanager +def authenticate(org, repo=None): + """ + A contextmanager that authenticates a user with GitHub via SSH if possible, otherwise via HTTPS. + + :param org: GitHub organisation to authenticate with + :type org: str + :param repo: GitHub repo (part of the org) to authenticate with. Default is the user's GitHub login. + :type repo: str, optional + :return: an authenticated user + :type: lib50.User + + Example usage:: + + from lib50 import authenticate + + with authenticate("me50") as user: + print(user.name) + + """ + with api.ProgressBar(_("Authenticating")) as progress_bar: + progress_bar.stop() + user = _authenticate_ssh(org, repo=repo) + if user is None: + # SSH auth failed, fallback to HTTPS + with _authenticate_https(org, repo=repo) as user: + yield user + else: + yield user + + +def logout(): + """ + Log out from git. + + :return: None + :type: None + """ + api._run(f"git credential-cache --socket {_CREDENTIAL_SOCKET} exit") + + +def _run_authenticated(user, command, quiet=False, timeout=None): + """Run a command, returns command output.""" + try: + with api._spawn(command, quiet, timeout) as child: + try: + child.expect(["Enter passphrase for key"]) + child.sendline(user.passphrase) + except pexpect.EOF: + pass + command_output = child.read().strip().replace("\r\n", "\n") + except pexpect.TIMEOUT: + api.logger.info(f"command {command} timed out") + raise TimeoutError(timeout) + + return command_output + + +def _authenticate_ssh(org, repo=None): + """Try authenticating via ssh, if succesful yields a User, otherwise raises Error.""" + + class State(enum.Enum): + FAIL = 0 + SUCCESS = 1 + PASSPHRASE_PROMPT = 2 + NEW_KEY = 3 + + # Require ssh-agent + child = pexpect.spawn("ssh -p443 -T git@ssh.github.com", encoding="utf8") + + # GitHub prints 'Hi {username}!...' when attempting to get shell access + try: + state = State(child.expect([ + "Permission denied", + "Hi (.+)! You've successfully authenticated", + "Enter passphrase for key", + "Are you sure you want to continue connecting" + ])) + except (pexpect.EOF, pexpect.TIMEOUT): + return None + + passphrase = "" + + try: + # New SSH connection + if state == State.NEW_KEY: + # yes to Continue connecting + child.sendline("yes") + + state = State(child.expect([ + "Permission denied", + "Hi (.+)! You've successfully authenticated", + "Enter passphrase for key" + ])) + + # If passphrase is needed, prompt and enter + if state == State.PASSPHRASE_PROMPT: + # Prompt passphrase + passphrase = _prompt_password("Enter passphrase for SSH key: ") + + # Enter passphrase + child.sendline(passphrase) + + state = State(child.expect([ + "Permission denied", + "Hi (.+)! You've successfully authenticated" + ])) + + # Succesfull authentication, done + if state == State.SUCCESS: + username = child.match.groups()[0] + # Failed authentication, nothing to be done + else: + return None + finally: + child.close() + + return User(name=username, + repo=f"ssh://git@ssh.github.com:443/{org}/{username if repo is None else repo}", + org=org, + passphrase=passphrase) + + +@contextlib.contextmanager +def _authenticate_https(org, repo=None): + """Try authenticating via HTTPS, if succesful yields User, otherwise raises Error.""" + _CREDENTIAL_SOCKET.parent.mkdir(mode=0o700, exist_ok=True) + try: + api.Git.cache = f"-c credential.helper= -c credential.helper='cache --socket {_CREDENTIAL_SOCKET}'" + git = api.Git().set(api.Git.cache) + + # Get credentials from cache if possible + with api._spawn(git("credential fill"), quiet=True) as child: + child.sendline("protocol=https") + child.sendline("host=github.com") + child.sendline("") + i = child.expect(["Username for '.+'", "Password for '.+'", + "username=([^\r]+)\r\npassword=([^\r]+)\r\n"]) + if i == 2: + username, password = child.match.groups() + else: + username = password = None + child.close() + child.exitstatus = 0 + + if password is None: + username = _prompt_username(_("GitHub username: ")) + password = _prompt_password(_("GitHub password: ")) + + # Credentials are correct, best cache them + with api._spawn(git("-c credentialcache.ignoresighup=true credential approve"), quiet=True) as child: + child.sendline("protocol=https") + child.sendline("host=github.com") + child.sendline(f"path={org}/{username}") + child.sendline(f"username={username}") + child.sendline(f"password={password}") + child.sendline("") + + yield User(name=username, + repo=f"https://{username}@github.com/{org}/{username if repo is None else repo}", + org=org) + except BaseException: + # Some error occured while this context manager is active, best forget credentials. + logout() + raise + + +def _prompt_username(prompt="Username: "): + """Prompt the user for username.""" + try: + while True: + username = input(prompt).strip() + if not username: + print("Username cannot be empty, please try again.") + elif "@" in username: + print("Please enter your GitHub username, not email.") + else: + return username + except EOFError: + print() + + +def _prompt_password(prompt="Password: "): + """Prompt the user for password, printing asterisks for each character""" + print(prompt, end="", flush=True) + password_bytes = [] + password_string = "" + + with _no_echo_stdin(): + while True: + # Read one byte + ch = sys.stdin.buffer.read(1)[0] + # If user presses Enter or ctrl-d + if ch in (ord("\r"), ord("\n"), 4): + print("\r") + break + # Del + elif ch == 127: + if len(password_string) > 0: + print("\b \b", end="", flush=True) + # Remove last char and its corresponding bytes + password_string = password_string[:-1] + password_bytes = list(password_string.encode("utf8")) + # Ctrl-c + elif ch == 3: + print("^C", end="", flush=True) + raise KeyboardInterrupt + else: + password_bytes.append(ch) + + # If byte added concludes a utf8 char, print * + try: + password_string = bytes(password_bytes).decode("utf8") + except UnicodeDecodeError: + pass + else: + print("*", end="", flush=True) + + if not password_string: + print("Password cannot be empty, please try again.") + return _prompt_password(prompt) + + return password_string + + +@contextlib.contextmanager +def _no_echo_stdin(): + """ + On Unix only, have stdin not echo input. + https://stackoverflow.com/questions/510357/python-read-a-single-character-from-the-user + """ + fd = sys.stdin.fileno() + old_settings = termios.tcgetattr(fd) + tty.setraw(fd) + try: + yield + finally: + termios.tcsetattr(fd, termios.TCSADRAIN, old_settings) \ No newline at end of file From f4bfb94e9a9f3373dafc216d4b0f624ea857bdbc Mon Sep 17 00:00:00 2001 From: Jelleas Date: Mon, 2 Aug 2021 18:26:40 +0200 Subject: [PATCH 56/64] authenticate.py -> authentication.py, fix tests --- lib50/_api.py | 3 ++- lib50/{authenticate.py => authentication.py} | 0 tests/api_tests.py | 13 +++++++------ 3 files changed, 9 insertions(+), 7 deletions(-) rename lib50/{authenticate.py => authentication.py} (100%) diff --git a/lib50/_api.py b/lib50/_api.py index 3dccdaf..b81fd41 100644 --- a/lib50/_api.py +++ b/lib50/_api.py @@ -12,6 +12,7 @@ import sys import tempfile import threading +import time import functools import attr @@ -22,7 +23,7 @@ from . import _, get_local_path from ._errors import * -from .authenticate import authenticate, logout, _run_authenticated +from .authentication import authenticate, logout, _run_authenticated from . import config as lib50_config __all__ = ["push", "local", "working_area", "files", "connect", diff --git a/lib50/authenticate.py b/lib50/authentication.py similarity index 100% rename from lib50/authenticate.py rename to lib50/authentication.py diff --git a/tests/api_tests.py b/tests/api_tests.py index 9e96b02..a1b842d 100644 --- a/tests/api_tests.py +++ b/tests/api_tests.py @@ -13,6 +13,7 @@ import pexpect import lib50._api +import lib50.authentication class TestGit(unittest.TestCase): def setUp(self): @@ -190,19 +191,19 @@ def mock_no_echo_stdin(self): def mock(): yield - old = lib50._api._no_echo_stdin + old = lib50.authentication._no_echo_stdin try: - lib50._api._no_echo_stdin = mock + lib50.authentication._no_echo_stdin = mock yield mock finally: - lib50._api._no_echo_stdin = old + lib50._api.authentication = old def test_ascii(self): f = io.StringIO() with self.mock_no_echo_stdin(), self.replace_stdin(), contextlib.redirect_stdout(f): sys.stdin.write(bytes("foo\n".encode("utf8"))) sys.stdin.seek(0) - password = lib50._api._prompt_password() + password = lib50.authentication._prompt_password() self.assertEqual(password, "foo") self.assertEqual(f.getvalue().count("*"), 3) @@ -212,7 +213,7 @@ def test_unicode(self): with self.mock_no_echo_stdin(), self.replace_stdin(), contextlib.redirect_stdout(f): sys.stdin.write(bytes("↔♣¾€\n".encode("utf8"))) sys.stdin.seek(0) - password = lib50._api._prompt_password() + password = lib50.authentication._prompt_password() self.assertEqual(password, "↔♣¾€") self.assertEqual(f.getvalue().count("*"), 4) @@ -229,7 +230,7 @@ def resolve_backspaces(str): with self.mock_no_echo_stdin(), self.replace_stdin(), contextlib.redirect_stdout(f): sys.stdin.write(bytes(f"↔{chr(127)}♣¾{chr(127)}€\n".encode("utf8"))) sys.stdin.seek(0) - password = lib50._api._prompt_password() + password = lib50.authentication._prompt_password() self.assertEqual(password, "♣€") self.assertEqual(resolve_backspaces(f.getvalue()).count("*"), 2) From 541d6d0c9238e9373e6daaea567d60da8f58503b Mon Sep 17 00:00:00 2001 From: Jelleas Date: Thu, 5 Aug 2021 12:18:19 +0200 Subject: [PATCH 57/64] show warning, fix hanging https, reprompt for ssh --- lib50/_api.py | 68 ++++++++++++++++++++--------------------- lib50/authentication.py | 57 +++++++++++++++++++++++----------- 2 files changed, 73 insertions(+), 52 deletions(-) diff --git a/lib50/_api.py b/lib50/_api.py index b81fd41..dbb9147 100644 --- a/lib50/_api.py +++ b/lib50/_api.py @@ -15,7 +15,6 @@ import time import functools -import attr import jellyfish import pexpect import requests @@ -23,7 +22,7 @@ from . import _, get_local_path from ._errors import * -from .authentication import authenticate, logout, _run_authenticated +from .authentication import authenticate, logout, run_authenticated from . import config as lib50_config __all__ = ["push", "local", "working_area", "files", "connect", @@ -134,24 +133,24 @@ def local(slug, offline=False, remove_origin=False, github_token=None): git = Git().set("-C {path}", path=str(local_path)) if not local_path.exists(): - _run(Git()("init {path}", path=str(local_path))) - _run(git(f"remote add origin {slug.origin}")) + run(Git()("init {path}", path=str(local_path))) + run(git(f"remote add origin {slug.origin}")) if not offline: # Get latest version of checks - _run(git("fetch origin --depth 1 {branch}", branch=slug.branch)) + run(git("fetch origin --depth 1 {branch}", branch=slug.branch)) # Tolerate checkout failure (e.g., when origin doesn't exist) try: - _run(git("checkout -f -B {branch} origin/{branch}", branch=slug.branch)) + run(git("checkout -f -B {branch} origin/{branch}", branch=slug.branch)) except Error: pass # Ensure that local copy of the repo is identical to remote copy - _run(git("reset --hard HEAD")) + run(git("reset --hard HEAD")) if remove_origin: - _run(git(f"remote remove origin")) + run(git(f"remote remove origin")) problem_path = (local_path / slug.problem).absolute() @@ -430,39 +429,38 @@ def prepare(tool, branch, user, included): try: clone_command = f"clone --bare --single-branch {user.repo} .git" try: - _run_authenticated(user, git.set(Git.cache)(f"{clone_command} --branch {branch}")) + run_authenticated(user, git.set(Git.cache)(f"{clone_command} --branch {branch}")) except Error: - _run_authenticated(user, git.set(Git.cache)(clone_command)) + run_authenticated(user, git.set(Git.cache)(clone_command)) except Error: - msg = _("Make sure your username and/or password are valid and {} is enabled for your account. To enable {}, ").format(tool, tool) + msg = _("Make sure your username and/or personal access token are valid and {} is enabled for your account. To enable {}, ").format(tool, tool) if user.org != DEFAULT_PUSH_ORG: msg += _("please contact your instructor.") else: msg += _("please go to {} in your web browser and try again.").format(AUTH_URL) - msg += _((" If you're using GitHub two-factor authentication, you'll need to create and use a personal access token " - "with the \"repo\" scope instead of your password. See https://cs50.ly/github-2fa for more information!")) + msg += _((" For instructions on how to set up a personal access token, please visit https://cs50.ly/github")) raise Error(msg) with ProgressBar(_("Preparing")) as progress_bar: - _run(git("config --bool core.bare false")) - _run(git("config --path core.worktree {area}", area=str(area))) + run(git("config --bool core.bare false")) + run(git("config --path core.worktree {area}", area=str(area))) try: - _run(git("checkout --force {branch} .gitattributes", branch=branch)) + run(git("checkout --force {branch} .gitattributes", branch=branch)) except Error: pass # Set user name/email in repo config - _run(git("config user.email {email}", email=user.email)) - _run(git("config user.name {name}", name=user.name)) + run(git("config user.email {email}", email=user.email)) + run(git("config user.name {name}", name=user.name)) # Switch to branch without checkout - _run(git("symbolic-ref HEAD {ref}", ref=f"refs/heads/{branch}")) + run(git("symbolic-ref HEAD {ref}", ref=f"refs/heads/{branch}")) # Git add all included files - _run(git(f"add -f {' '.join(shlex.quote(f) for f in included)}")) + run(git(f"add -f {' '.join(shlex.quote(f) for f in included)}")) # Remove gitattributes from included if Path(".gitattributes").exists() and ".gitattributes" in included: @@ -510,9 +508,9 @@ def upload(branch, user, tool, data): # Commit + push git = Git().set(Git.working_area) - _run(git("commit -m {msg} --allow-empty", msg=commit_message)) - _run_authenticated(user, git.set(Git.cache)("push origin {branch}", branch=branch)) - commit_hash = _run(git("rev-parse HEAD")) + run(git("commit -m {msg} --allow-empty", msg=commit_message)) + run_authenticated(user, git.set(Git.cache)("push origin {branch}", branch=branch)) + commit_hash = run(git("rev-parse HEAD")) return user.name, commit_hash @@ -619,7 +617,7 @@ def get_local_slugs(tool, similar_to=""): org, repo = path.parts[0:2] if (org, repo) not in branch_map: git = Git().set("-C {path}", path=str(local_path / path.parent)) - branch = _run(git("rev-parse --abbrev-ref HEAD")) + branch = run(git("rev-parse --abbrev-ref HEAD")) branch_map[(org, repo)] = branch # Reconstruct slugs for each config file @@ -795,11 +793,11 @@ def _get_branches(self): """Get branches from org/repo.""" if self.offline: local_path = get_local_path() / self.org / self.repo - output = _run(f"git -C {shlex.quote(str(local_path))} show-ref --heads").split("\n") + output = run(f"git -C {shlex.quote(str(local_path))} show-ref --heads").split("\n") else: cmd = f"git ls-remote --heads {self.origin}" try: - with _spawn(cmd, timeout=3) as child: + with spawn(cmd, timeout=3) as child: output = child.read().strip().split("\r\n") except pexpect.TIMEOUT: if "Username for" in child.buffer: @@ -905,7 +903,7 @@ def flush(self): @contextlib.contextmanager -def _spawn(command, quiet=False, timeout=None): +def spawn(command, quiet=False, timeout=None): """Run (spawn) a command with `pexpect.spawn`""" # Spawn command child = pexpect.spawn( @@ -934,10 +932,10 @@ def _spawn(command, quiet=False, timeout=None): raise Error() -def _run(command, quiet=False, timeout=None): +def run(command, quiet=False, timeout=None): """Run a command, returns command output.""" try: - with _spawn(command, quiet, timeout) as child: + with spawn(command, quiet, timeout) as child: command_output = child.read().strip().replace("\r\n", "\n") except pexpect.TIMEOUT: logger.info(f"command {command} timed out") @@ -1060,17 +1058,17 @@ def _lfs_add(files, git): "and then re-run!").format("\n".join(larges))) # Install git-lfs for this repo - _run(git("lfs install --local")) + run(git("lfs install --local")) # For pre-push hook - _run(git("config credential.helper cache")) + run(git("config credential.helper cache")) # Rm previously added file, have lfs track file, add file again for large in larges: - _run(git("rm --cached {large}", large=large)) - _run(git("lfs track {large}", large=large)) - _run(git("add {large}", large=large)) - _run(git("add --force .gitattributes")) + run(git("rm --cached {large}", large=large)) + run(git("lfs track {large}", large=large)) + run(git("add {large}", large=large)) + run(git("add --force .gitattributes")) def _is_relative_to(path, *others): diff --git a/lib50/authentication.py b/lib50/authentication.py index ab2f376..643fd79 100644 --- a/lib50/authentication.py +++ b/lib50/authentication.py @@ -1,17 +1,19 @@ +import attr import contextlib import enum import pexpect import sys +import termcolor import termios import tty from pathlib import Path -import attr - from . import _ from . import _api as api +from ._errors import ConnectionError +__all__ = ["User", "authenticate", "logout"] _CREDENTIAL_SOCKET = Path("~/.git-credential-cache/lib50").expanduser() @@ -22,7 +24,7 @@ class User: name = attr.ib() repo = attr.ib() org = attr.ib() - passphrase = attr.ib() + passphrase = attr.ib(default=str) email = attr.ib(default=attr.Factory(lambda self: f"{self.name}@users.noreply.github.com", takes_self=True), init=False) @@ -48,12 +50,17 @@ def authenticate(org, repo=None): """ with api.ProgressBar(_("Authenticating")) as progress_bar: + # Both authentication methods can require user input, best stop the bar progress_bar.stop() + + # Try auth through SSH user = _authenticate_ssh(org, repo=repo) + + # SSH aut failed, fallback to HTTPS if user is None: - # SSH auth failed, fallback to HTTPS with _authenticate_https(org, repo=repo) as user: yield user + # yield SSH user else: yield user @@ -65,19 +72,29 @@ def logout(): :return: None :type: None """ - api._run(f"git credential-cache --socket {_CREDENTIAL_SOCKET} exit") + api.run(f"git credential-cache --socket {_CREDENTIAL_SOCKET} exit") -def _run_authenticated(user, command, quiet=False, timeout=None): - """Run a command, returns command output.""" +def run_authenticated(user, command, quiet=False, timeout=None): + """Run a command as a authenticated user. Returns command output.""" try: - with api._spawn(command, quiet, timeout) as child: - try: - child.expect(["Enter passphrase for key"]) + with api.spawn(command, quiet, timeout) as child: + match = child.expect([ + "Enter passphrase for key", + "Password for", + pexpect.EOF + ]) + + # In case "Enter passphrase for key" appears, send user's passphrase + if match == 0: child.sendline(user.passphrase) - except pexpect.EOF: pass + # In case "Password for" appears, https authentication failed + elif match == 1: + raise ConnectionError + command_output = child.read().strip().replace("\r\n", "\n") + except pexpect.TIMEOUT: api.logger.info(f"command {command} timed out") raise TimeoutError(timeout) @@ -122,8 +139,8 @@ class State(enum.Enum): "Enter passphrase for key" ])) - # If passphrase is needed, prompt and enter - if state == State.PASSPHRASE_PROMPT: + # while passphrase is needed, prompt and enter + while state == State.PASSPHRASE_PROMPT: # Prompt passphrase passphrase = _prompt_password("Enter passphrase for SSH key: ") @@ -132,7 +149,8 @@ class State(enum.Enum): state = State(child.expect([ "Permission denied", - "Hi (.+)! You've successfully authenticated" + "Hi (.+)! You've successfully authenticated", + "Enter passphrase for key" ])) # Succesfull authentication, done @@ -159,7 +177,7 @@ def _authenticate_https(org, repo=None): git = api.Git().set(api.Git.cache) # Get credentials from cache if possible - with api._spawn(git("credential fill"), quiet=True) as child: + with api.spawn(git("credential fill"), quiet=True) as child: child.sendline("protocol=https") child.sendline("host=github.com") child.sendline("") @@ -174,10 +192,10 @@ def _authenticate_https(org, repo=None): if password is None: username = _prompt_username(_("GitHub username: ")) - password = _prompt_password(_("GitHub password: ")) + password = _prompt_password(_("GitHub Personal Access Token: ")) # Credentials are correct, best cache them - with api._spawn(git("-c credentialcache.ignoresighup=true credential approve"), quiet=True) as child: + with api.spawn(git("-c credentialcache.ignoresighup=true credential approve"), quiet=True) as child: child.sendline("protocol=https") child.sendline("host=github.com") child.sendline(f"path={org}/{username}") @@ -189,6 +207,11 @@ def _authenticate_https(org, repo=None): repo=f"https://{username}@github.com/{org}/{username if repo is None else repo}", org=org) except BaseException: + msg = _("You might be using your GitHub password to log in," \ + " but that's no longer possible. But you can still use" \ + " check50 and submit50! See https://cs50.ly/github for instructions.") + api.logger.warning(termcolor.colored(msg, attrs=["bold"])) + # Some error occured while this context manager is active, best forget credentials. logout() raise From 9e768eed10bdfd6040ff65003b7872515f2e44a8 Mon Sep 17 00:00:00 2001 From: Jelleas Date: Thu, 5 Aug 2021 12:21:05 +0200 Subject: [PATCH 58/64] rm undefined and unused variable --- lib50/_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib50/_api.py b/lib50/_api.py index dbb9147..a3a7192 100644 --- a/lib50/_api.py +++ b/lib50/_api.py @@ -1047,7 +1047,7 @@ def _lfs_add(files, git): if huges: raise Error(_("These files are too large to be submitted:\n{}\n" "Remove these files from your directory " - "and then re-run!").format("\n".join(huges), org)) + "and then re-run!").format("\n".join(huges))) # Add large files (>100MB) with git-lfs if larges: From 0531d30549381c027d9be5183392db51f9f0ed37 Mon Sep 17 00:00:00 2001 From: Jelleas Date: Thu, 5 Aug 2021 14:03:06 +0200 Subject: [PATCH 59/64] formatting --- lib50/authentication.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib50/authentication.py b/lib50/authentication.py index 643fd79..f433c65 100644 --- a/lib50/authentication.py +++ b/lib50/authentication.py @@ -181,8 +181,11 @@ def _authenticate_https(org, repo=None): child.sendline("protocol=https") child.sendline("host=github.com") child.sendline("") - i = child.expect(["Username for '.+'", "Password for '.+'", - "username=([^\r]+)\r\npassword=([^\r]+)\r\n"]) + i = child.expect([ + "Username for '.+'", + "Password for '.+'", + "username=([^\r]+)\r\npassword=([^\r]+)\r\n" + ]) if i == 2: username, password = child.match.groups() else: From fb74d79d09f59d262fcfb69abed69de24944890a Mon Sep 17 00:00:00 2001 From: Jelleas Date: Fri, 6 Aug 2021 12:05:48 +0200 Subject: [PATCH 60/64] only show GitHub password warning on normal exceptions --- lib50/authentication.py | 57 ++++++++++++++++++++++------------------- 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/lib50/authentication.py b/lib50/authentication.py index f433c65..15702ac 100644 --- a/lib50/authentication.py +++ b/lib50/authentication.py @@ -172,31 +172,32 @@ class State(enum.Enum): def _authenticate_https(org, repo=None): """Try authenticating via HTTPS, if succesful yields User, otherwise raises Error.""" _CREDENTIAL_SOCKET.parent.mkdir(mode=0o700, exist_ok=True) - try: - api.Git.cache = f"-c credential.helper= -c credential.helper='cache --socket {_CREDENTIAL_SOCKET}'" - git = api.Git().set(api.Git.cache) - - # Get credentials from cache if possible - with api.spawn(git("credential fill"), quiet=True) as child: - child.sendline("protocol=https") - child.sendline("host=github.com") - child.sendline("") - i = child.expect([ - "Username for '.+'", - "Password for '.+'", - "username=([^\r]+)\r\npassword=([^\r]+)\r\n" - ]) - if i == 2: - username, password = child.match.groups() - else: - username = password = None - child.close() - child.exitstatus = 0 + api.Git.cache = f"-c credential.helper= -c credential.helper='cache --socket {_CREDENTIAL_SOCKET}'" + git = api.Git().set(api.Git.cache) + + # Get credentials from cache if possible + with api.spawn(git("credential fill"), quiet=True) as child: + child.sendline("protocol=https") + child.sendline("host=github.com") + child.sendline("") + i = child.expect([ + "Username for '.+'", + "Password for '.+'", + "username=([^\r]+)\r\npassword=([^\r]+)\r\n" + ]) + if i == 2: + username, password = child.match.groups() + else: + username = password = None + child.close() + child.exitstatus = 0 - if password is None: - username = _prompt_username(_("GitHub username: ")) - password = _prompt_password(_("GitHub Personal Access Token: ")) + # If password is not in cache, prompt + if password is None: + username = _prompt_username(_("GitHub username: ")) + password = _prompt_password(_("GitHub Personal Access Token: ")) + try: # Credentials are correct, best cache them with api.spawn(git("-c credentialcache.ignoresighup=true credential approve"), quiet=True) as child: child.sendline("protocol=https") @@ -209,15 +210,19 @@ def _authenticate_https(org, repo=None): yield User(name=username, repo=f"https://{username}@github.com/{org}/{username if repo is None else repo}", org=org) - except BaseException: + except Exception: msg = _("You might be using your GitHub password to log in," \ - " but that's no longer possible. But you can still use" \ - " check50 and submit50! See https://cs50.ly/github for instructions.") + " but that's no longer possible. But you can still use" \ + " check50 and submit50! See https://cs50.ly/github for instructions.") api.logger.warning(termcolor.colored(msg, attrs=["bold"])) # Some error occured while this context manager is active, best forget credentials. logout() raise + except BaseException: + # Some special error (like SIGINT) occured while this context manager is active, best forget credentials. + logout() + raise def _prompt_username(prompt="Username: "): From e4a22c217d12e366edacbb71cd1744f3a95bbd34 Mon Sep 17 00:00:00 2001 From: Jelleas Date: Fri, 6 Aug 2021 12:21:35 +0200 Subject: [PATCH 61/64] warnings for passphrase re-prompt --- lib50/authentication.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lib50/authentication.py b/lib50/authentication.py index 15702ac..4ece6c4 100644 --- a/lib50/authentication.py +++ b/lib50/authentication.py @@ -153,6 +153,17 @@ class State(enum.Enum): "Enter passphrase for key" ])) + # In case of a re-prompt, warn the user + if state == State.PASSPHRASE_PROMPT: + print("Looks like that passphrase is incorrect, please try again.") + + # In case of failed auth and no re-prompt, warn user and fall back on https + if state == State.FAIL: + print("Looks like that passphrase is incorrect, trying authentication through"\ + " username and Personal Access Token instead.") + api.logger.warning("See https://cs50.ly/github for instructions on"\ + " the different authentication methods if you haven't already!") + # Succesfull authentication, done if state == State.SUCCESS: username = child.match.groups()[0] From 15a3a8246f5c90616e39fbf88c73574274ae6e04 Mon Sep 17 00:00:00 2001 From: Jelleas Date: Fri, 6 Aug 2021 12:28:47 +0200 Subject: [PATCH 62/64] through -> with --- lib50/authentication.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib50/authentication.py b/lib50/authentication.py index 4ece6c4..4627280 100644 --- a/lib50/authentication.py +++ b/lib50/authentication.py @@ -159,7 +159,7 @@ class State(enum.Enum): # In case of failed auth and no re-prompt, warn user and fall back on https if state == State.FAIL: - print("Looks like that passphrase is incorrect, trying authentication through"\ + print("Looks like that passphrase is incorrect, trying authentication with"\ " username and Personal Access Token instead.") api.logger.warning("See https://cs50.ly/github for instructions on"\ " the different authentication methods if you haven't already!") From 6037838143e4a398d93cba4dbff7b13437a41c83 Mon Sep 17 00:00:00 2001 From: Jelleas Date: Fri, 6 Aug 2021 12:53:46 +0200 Subject: [PATCH 63/64] preemptive warning --- lib50/authentication.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/lib50/authentication.py b/lib50/authentication.py index 4627280..f180461 100644 --- a/lib50/authentication.py +++ b/lib50/authentication.py @@ -53,10 +53,16 @@ def authenticate(org, repo=None): # Both authentication methods can require user input, best stop the bar progress_bar.stop() + # Show a quick reminder to check https://cs50.ly/github + warning = "GitHub now requires that you use SSH or a personal access token"\ + " instead of a password to log in, but you can still use check50 and submit50!"\ + " See https://cs50.ly/github for instructions if you haven't already!" + api.logger.warning(termcolor.colored(warning, attrs=["bold"])) + # Try auth through SSH user = _authenticate_ssh(org, repo=repo) - # SSH aut failed, fallback to HTTPS + # SSH auth failed, fallback to HTTPS if user is None: with _authenticate_https(org, repo=repo) as user: yield user @@ -161,8 +167,10 @@ class State(enum.Enum): if state == State.FAIL: print("Looks like that passphrase is incorrect, trying authentication with"\ " username and Personal Access Token instead.") - api.logger.warning("See https://cs50.ly/github for instructions on"\ - " the different authentication methods if you haven't already!") + + warning = "See https://cs50.ly/github for instructions on"\ + " the different authentication methods if you haven't already!" + api.logger.warning(termcolor.colored(warning, attrs=["bold"])) # Succesfull authentication, done if state == State.SUCCESS: From 2bea4e4d13e3cc5ab090474a5671d3f5329ec874 Mon Sep 17 00:00:00 2001 From: Jelleas Date: Sat, 7 Aug 2021 12:32:00 +0200 Subject: [PATCH 64/64] https auth phrasing + api.logger.warning => print --- lib50/authentication.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib50/authentication.py b/lib50/authentication.py index f180461..934ccc7 100644 --- a/lib50/authentication.py +++ b/lib50/authentication.py @@ -57,7 +57,7 @@ def authenticate(org, repo=None): warning = "GitHub now requires that you use SSH or a personal access token"\ " instead of a password to log in, but you can still use check50 and submit50!"\ " See https://cs50.ly/github for instructions if you haven't already!" - api.logger.warning(termcolor.colored(warning, attrs=["bold"])) + print(termcolor.colored(warning, color="yellow", attrs=["bold"])) # Try auth through SSH user = _authenticate_ssh(org, repo=repo) @@ -170,7 +170,7 @@ class State(enum.Enum): warning = "See https://cs50.ly/github for instructions on"\ " the different authentication methods if you haven't already!" - api.logger.warning(termcolor.colored(warning, attrs=["bold"])) + print(termcolor.colored(warning, color="yellow", attrs=["bold"])) # Succesfull authentication, done if state == State.SUCCESS: @@ -213,8 +213,8 @@ def _authenticate_https(org, repo=None): # If password is not in cache, prompt if password is None: - username = _prompt_username(_("GitHub username: ")) - password = _prompt_password(_("GitHub Personal Access Token: ")) + username = _prompt_username(_("Enter username for GitHub: ")) + password = _prompt_password(_("Enter personal access token for GitHub: ")) try: # Credentials are correct, best cache them @@ -233,7 +233,7 @@ def _authenticate_https(org, repo=None): msg = _("You might be using your GitHub password to log in," \ " but that's no longer possible. But you can still use" \ " check50 and submit50! See https://cs50.ly/github for instructions.") - api.logger.warning(termcolor.colored(msg, attrs=["bold"])) + print(termcolor.colored(msg, color="yellow", attrs=["bold"])) # Some error occured while this context manager is active, best forget credentials. logout()