diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 54d1122..cb06acf 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,34 +19,48 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, macos-latest] - python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, "3.10", pypy2, pypy3] + os: [ubuntu-22.04, macos-latest] + python-version: [3.8, 3.9, "3.10", "3.11", "3.12", "pypy3.9"] include: + - os: ubuntu-22.04 + python-version: "2.7" - os: windows-latest python-version: 3.9 - os: windows-latest python-version: "3.10" + - os: windows-latest + python-version: "3.11" + - os: windows-latest + python-version: "3.12" exclude: - os: macos-latest python-version: 3.5 - os: macos-latest python-version: 3.6 - os: macos-latest - python-version: pypy2 - - os: macos-latest - python-version: pypy3 + python-version: "pypy3.9" steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} via setup-python + if: matrix.python-version != '2.7' + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} + - name: Set up Python ${{ matrix.python-version }} via apt-get + if: matrix.python-version == '2.7' + run: | + set -eux + sudo apt-get update + sudo apt-get install -y python2 python3-virtualenv + virtualenv -p python2 "${{ runner.temp }}/venv" + echo "${{ runner.temp }}/venv/bin" >> $GITHUB_PATH - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install coverage flake8 + # must install all dependencies so the tab modules can be generated + python -m pip install coverage flake8 ply setuptools python -m pip install -e . - name: Lint with flake8 run: | @@ -55,9 +69,13 @@ jobs: run: | python -OO -m unittest calmjs.parse.tests.make_suite coverage run --include=src/* -m unittest calmjs.parse.tests.make_suite + # Python 3.12 on Windows resulted in MemoryError here, so optional. + - name: Coverage report + run: | coverage report -m + continue-on-error: true - name: Coveralls - if: ${{ matrix.os == 'ubuntu-latest' && matrix.python-version != '2.7' && matrix.python-version != 'pypy2' }} + if: ${{ matrix.os == 'ubuntu-22.04' && matrix.python-version != '2.7' && matrix.python-version != 'pypy2' }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | diff --git a/CHANGES.rst b/CHANGES.rst index 91e1220..ed62ad7 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,16 @@ Changelog ========= +1.3.1 - 2023-10-28 +------------------ + +- Modified existing ``setup.py`` hook from an install hook to a build + hook to ensure the generated module files are present. Should any of + those modules are missing and the required dependencies for are not + present (i.e. ``ply`` and ``setuptools``), the build will result in a + non-zero exit status and the documented error message should reflect + which of the required dependencies are missing. + 1.3.0 - 2021-10-08 ------------------ diff --git a/MANIFEST.in b/MANIFEST.in index 1e14990..562a1ed 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -6,3 +6,7 @@ include *.rst include LICENSE recursive-exclude src *.pyc recursive-exclude src *.pyo +prune .github +exclude .gitignore +exclude appveyor.yml +exclude .flake8 diff --git a/README.rst b/README.rst index bfd0fd9..5407625 100644 --- a/README.rst +++ b/README.rst @@ -78,9 +78,8 @@ As this package uses |ply|, it requires the generation of optimization modules for its lexer. The wheel distribution of |calmjs.parse| does not require this extra step as it contains these pre-generated modules for |ply| up to version 3.11 (the latest version available at the time -of previous release), however the source tarball or if |ply| version -that is installed lies outside of the supported versions, the following -caveats will apply. +of previous release), however the version of |ply| that is installed is +beyond the supported version, the following caveats will apply. If a more recent release of |ply| becomes available and the environment upgrades to that version, those pre-generated modules may become @@ -89,11 +88,18 @@ A corrective action can be achieved through a `manual optimization`_ step if a newer version of |calmjs.parse| is not available, or |ply| may be downgraded back to version 3.11 if possible. +Alternatively, install a more recent version of |calmjs.parse| wheel +that has the most complete set of pre-generated modules built. + Once the package is installed, the installation may be `tested`_ or be `used directly`_. -Alternative installation methods (for developers, advanced users) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Manual installation and packaging requirements +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +*This section is for developers and advanced users; contains important +information for package maintainers for OS distributions (e.g. Linux) +that will prevent less than ideal experiences for downstream users.* Development is still ongoing with |calmjs.parse|, for the latest features and bug fixes, the development version may be installed through @@ -101,14 +107,58 @@ git like so: .. code:: console - $ pip install git+https://github.com/calmjs/calmjs.parse.git#egg=calmjs.parse + $ pip install ply setuptools # this MUST be done first; see below for reason + $ pip install -e git+https://github.com/calmjs/calmjs.parse.git#egg=calmjs.parse -Alternatively, the git repository can be cloned directly and execute -``python setup.py develop`` while inside the root of the source -directory. +Note that all dependencies MUST be pre-installed ``setup.py build`` step +to run, otherwise the build step required to create the pre-generated +modules will result in failure. + +If |ply| isn't installed: + +.. code:: console + + $ python -m pip install -e . + ... + running egg_info + ... + WARNING: cannot find distribution for 'ply'; using default value, + assuming 'ply==3.11' for pre-generated modules + ERROR: cannot find pre-generated modules for the assumed 'ply' + version from above and/or cannot `import ply` to build generated + modules, aborting build; please either ensure that the source + archive containing the pre-generate modules is being used, or that + the python package 'ply' is installed and available for import + before attempting to use the setup.py to build this package; please + refer to the top level README for further details + +If ``setuptools`` isn't installed: + +.. code:: console -A manual optimization step may need to be performed for platforms and -systems that do not have utf8 as their default encoding. + $ python -m pip install -e . + ... + running egg_info + ... + Traceback (most recent call last): + ... + ModuleNotFoundError: No module named 'pkg_resources' + +Naturally, the git repository can be cloned directly and execute +``python setup.py develop`` while inside the root of the source +directory; again, both |ply| AND ``setuptools`` MUST already have be +available for import. + +As the git repository does NOT contain any pre-generated modules or +code, the above message is likely to be seen by developers or distro +maintainers who are on their first try at interacting with this +software. However, the zip archives released on PyPI starting from +version 1.3.0 do contain these modules fully pre-generated, thus they +may be used as part of a standard installation step, i.e. without +requiring |ply| be available for import before usage of the ``setup.py`` +for any purpose. While the same warning message about |ply| being +missing may be shown, the pre-generated modules will allow the build +step to proceed as normal. Manual optimization ~~~~~~~~~~~~~~~~~~~ @@ -714,6 +764,7 @@ Object assignments from a given script file: Further details and example usage can be consulted from the various docstrings found within the module. + Limitations ----------- @@ -735,6 +786,7 @@ comments. Likewise, any comments before the ``:`` token in a ternary statement will also be discarded as that is the second token consumed by the production rule that produces a ``Conditional`` node. + Troubleshooting --------------- @@ -768,6 +820,59 @@ A workaround helper script is provided, it may be executed like so: Further details on this topic may be found in the `manual optimization`_ section of this document. +WARNING: There are unused tokens on import +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This indicates that the installation method or source for this package +being imported isn't optimized. A quick workaround is to follow the +instructions at the `manual optimization`_ section of this document to +ensure these messages are no longer generated (and if this warning +happens every time the module is imported, it means the symbol tables +are regenerated every time that happens and this extra computational +overhead should be corrected through the generation of that optimization +module). + +The optimization modules are included with the wheel release and the +source release on PyPI, but it is not part of the source repository as +generated code are never committed. Should a binary release made by +a third-party results in this warning upon import, their release should +be corrected to include the optimization module. + +Moreover, there are safeguards in place that prevent this warning from +being generated for releases made for releases from 1.3.1 onwards by +a more heavy handed enforcement of this optimization step at build time, +but persistent (or careless) actors may circumvent this during the build +process, but official releases made through PyPI should include the +required optimization for all supported |ply| versions (which are +versions 3.6 to 3.11, inclusive). + +Alternatively, this issue may also occur via usage of ``pyinstaller`` +if the package metadata is not copied for |ply| in versions prior to +``calmjs.parse-1.3.1`` and will always occur if the hidden imports are +not declared for those optimization modules. The following hook should +may be used to ensure |calmjs.parse| functions correctly in the compiled +binary: + +.. code:: python + + from PyInstaller.utils.hooks import collect_data_files, copy_metadata + from calmjs.parse.utils import generate_tab_names + + datas = [] + datas.extend(collect_data_files("ply")) + datas.extend(copy_metadata("ply")) + datas.extend(collect_data_files("calmjs.parse")) + datas.extend(copy_metadata("calmjs.parse")) + + hiddenimports = [] + hiddenimports.extend(generate_tab_names('calmjs.parse.parsers.es5')) + + # if running under Python 3 with ply-3.11, above is equivalent to + # hiddenimports = [ + # "calmjs.parse.parsers.lextab_es5_py3_ply3_11", + # "calmjs.parse.parsers.yacctab_es5_py3_ply3_11", + # ] + Slow performance ~~~~~~~~~~~~~~~~ @@ -781,6 +886,18 @@ this will may require both the token and layout functions not having arguments with name collisions, and the new function will take in all of those arguments in one go. +ERROR message about import error when trying to install +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +As noted in the error message, the |ply|_ and ``setuptools`` package +must be installed before attempting to install build the package in the +situation where the pre-generated modules are missing. This situation +may be caused by building directly using the source provided by the +source code repository, or where there is no matching pre-generated +module matching with the installed version of |ply|. Please ensure that +|ply| is installed and available first before installing from source if +this error message is sighted. + Contribute ---------- diff --git a/appveyor.yml b/appveyor.yml index 3e8bbef..cda51f6 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,24 +1,21 @@ +image: Visual Studio 2022 + environment: matrix: - PYTHON: "C:\\Python27" - nodejs_version: "4.6" - PYTHON: "C:\\Python33" - nodejs_version: "4.6" - PYTHON: "C:\\Python34" - nodejs_version: "6.9" - PYTHON: "C:\\Python35" - nodejs_version: "6.9" - PYTHON: "C:\\Python36" - nodejs_version: "8" - PYTHON: "C:\\Python37" - nodejs_version: "10" - PYTHON: "C:\\Python38" - nodejs_version: "10" + - PYTHON: "C:\\Python39-x64" + - PYTHON: "C:\\Python310-x64" + - PYTHON: "C:\\Python311-x64" install: - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" - - ps: Install-Product node $env:nodejs_version - - "%PYTHON%\\python.exe -m pip install coverage" + - "%PYTHON%\\python.exe -m pip install setuptools coverage ply" - "%PYTHON%\\python.exe setup.py install" test_script: diff --git a/setup.py b/setup.py index 2535048..4a344e6 100644 --- a/setup.py +++ b/setup.py @@ -1,16 +1,22 @@ -import atexit +import os import sys from setuptools import setup, find_packages -from setuptools.command.install import install +from setuptools.command.build_py import build_py from subprocess import call -class InstallHook(install): - """For hooking the optimizer when setup exits""" +class BuildHook(build_py): + """Forcing the optimizer to run before the build step""" def __init__(self, *a, **kw): - install.__init__(self, *a, **kw) - atexit.register( - call, [sys.executable, '-m', 'calmjs.parse.parsers.optimize']) + # must use clone of this, otherwise Python on Windows gets sad. + env = os.environ.copy() + env['PYTHONPATH'] = 'src' + code = call([ + sys.executable, '-m', 'calmjs.parse.parsers.optimize', '--build' + ], env=env) + if code: + sys.exit(1) + build_py.__init__(self, *a, **kw) # Attributes @@ -32,6 +38,8 @@ def __init__(self, *a, **kw): Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 +Programming Language :: Python :: 3.11 +Programming Language :: Python :: 3.12 """.strip().splitlines() long_description = ( @@ -59,7 +67,7 @@ def __init__(self, *a, **kw): include_package_data=True, zip_safe=False, cmdclass={ - 'install': InstallHook, + 'build_py': BuildHook, }, install_requires=[ 'setuptools', diff --git a/src/calmjs/parse/__init__.py b/src/calmjs/parse/__init__.py index db42509..72c4f30 100644 --- a/src/calmjs/parse/__init__.py +++ b/src/calmjs/parse/__init__.py @@ -3,7 +3,14 @@ Quick access helper functions """ -from calmjs.parse.factory import ParserUnparserFactory +try: + from calmjs.parse.factory import ParserUnparserFactory +except ImportError as e: # pragma: no cover + exc = e + def import_error(*a, **kw): + raise exc -es5 = ParserUnparserFactory('es5', 'pretty_print', 'minify_print') + es5 = import_error +else: + es5 = ParserUnparserFactory('es5', 'pretty_print', 'minify_print') diff --git a/src/calmjs/parse/parsers/es5.py b/src/calmjs/parse/parsers/es5.py index f2db89c..042510c 100644 --- a/src/calmjs/parse/parsers/es5.py +++ b/src/calmjs/parse/parsers/es5.py @@ -42,8 +42,10 @@ asttypes = AstTypesFactory(pretty_print, ReprWalker()) -# The default values for the `Parser` constructor, passed on to ply; they must -# be strings +# These default values for the `Parser` constructor, passed on to ply; +# they must be strings; these values are for reference only as +# modifications to this value will not change what's been set up as +# the Parser's default. lextab, yacctab = generate_tab_names(__name__) diff --git a/src/calmjs/parse/parsers/optimize.py b/src/calmjs/parse/parsers/optimize.py index af01282..c7e8a96 100644 --- a/src/calmjs/parse/parsers/optimize.py +++ b/src/calmjs/parse/parsers/optimize.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -Helpers that will forcibly regenerate the tab files. +Helpers for maintenance/generation of the lextab/yacctab modules. The original goal of this was to force the creation of tab files using the utf8 codec to workaround issues with the ply package, for systems @@ -8,19 +8,36 @@ """ import codecs +import os import sys from functools import partial from os import unlink from os.path import exists -from ply import lex from importlib import import_module +from calmjs.parse.utils import generate_tab_names +from calmjs.parse.utils import ply_dist -# have to do this for every parser modules -from calmjs.parse.parsers import es5 +_ASSUME_PLY_VERSION = '3.11' +_ASSUME_ENVVAR = 'CALMJS_PARSE_ASSUME_PLY_VERSION' -def purge_tabs(module): +def validate_imports(*imports): paths = [] + missing = [] + for name in imports: + try: + import_module(name) + except ImportError: + missing.append(name) + else: + paths.append(sys.modules.pop(name).__file__) + return paths, missing + + +def find_tab_paths(module): + # return a list of lextab/yacctab module paths and a list of missing + # import names. + names = [] for entry in ('lextab', 'yacctab'): # we assume the specified entries are defined as such name = getattr(module, entry) @@ -31,14 +48,12 @@ def purge_tabs(module): 'provided module `%s` does not export expected tab values ' % module.__name__ ) - try: - import_module(name) - except ImportError: - # don't need to do anything - pass - else: - paths.append(sys.modules.pop(name).__file__) + names.append(name) + return validate_imports(*names) + +def purge_tabs(module): + paths, _ = find_tab_paths(module) unlink_modules(verify_paths(paths)) @@ -46,10 +61,15 @@ def verify_paths(paths): for path in paths: if exists(path): yield path + # locate any adjacent .py[co]? files based on module path + # returned; mostly a problem with Python 2 if path[-4:] in ('.pyc', '.pyo'): - # find the .py file, too. if exists(path[:-1]): yield path[:-1] + else: + for c in 'co': + if exists(path + c): + yield path + c def unlink_modules(paths): @@ -64,13 +84,118 @@ def reoptimize(module): module.Parser() -def reoptimize_all(monkey_patch=False): +def _assume_ply_version(): + version = os.environ.get(_ASSUME_ENVVAR, _ASSUME_PLY_VERSION) + if ply_dist is None: + if _ASSUME_ENVVAR in os.environ: + source = "using environment variable %r" % _ASSUME_ENVVAR + else: + # allow bypassing of setuptools as ply provides this + # attribute + try: + import ply + version = ply.__version__ + source = "using value provided by ply" + except ImportError: # pragma: no cover + ply = None + source = "using default value" + + sys.stderr.write( + u"WARNING: cannot find distribution for 'ply'; " + "%s, assuming 'ply==%s' for pre-generated modules\n" % ( + source, version)) + return version + + +def optimize_build(module_name, assume_ply_version=True): + """ + optimize build helper for first build + + assume_ply_version + This flag denotes whether or not to assume a ply version should + ply be NOT installed; this will either assume ply to be whatever + value assigned to _ASSUME_PLY_VERSION (i.e. 3.11), or read from + the environment variable `CALMJS_PARSE_ASSUME_PLY_VERSION`. + + The goal is to allow the build to proceed if the pre-generated + files are already present, before the dependency resolution at + the installation time actually kicks in to install ply. + + Default: True + """ + + kws = {} + if assume_ply_version: + kws['_version'] = _assume_ply_version() + + lextab, yacctab = generate_tab_names(module_name, **kws) + paths, missing = validate_imports(lextab, yacctab) + if missing: + # only import, purge and regenerate if any are missing. + unlink_modules(verify_paths(paths)) + module = import_module(module_name) + # use whatever assumed version or otherwise as set up by + # the local generation function. + module.Parser(lextab=lextab, yacctab=yacctab) + + +def reoptimize_all(monkey_patch=False, first_build=False): + """ + The main optimize method for maintainence of the generated tab + modules required by ply + + Arguments: + + monkey_patch + patches the default open function in ply.lex to use utf8 + + default: False + + first_build + flag for switching between reoptimize/optimize_build method; + setting the flag to True specifies the latter. + + default: False + """ + if monkey_patch: - lex.open = partial(codecs.open, encoding='utf8') - modules = (es5,) - for module in modules: - reoptimize(module) + try: + from ply import lex + except ImportError: # pragma: no cover + pass # fail later; only fail if import ply is truly needed + else: + lex.open = partial(codecs.open, encoding='utf8') + + modules = ('.es5',) + try: + for name in modules: + if first_build: + # A consideration for modifying this flag to simply + # check for a marker file to denote none of this being + # needed (i.e. this tarball was fully prepared), but it + # will not solve the issue where the distro packager + # already got an even more recent version of ply + # installed (as unlikely as that is) and that build step + # then is completely skipped. + optimize_build('calmjs.parse.parsers' + name) + else: + module = import_module(name, 'calmjs.parse.parsers') + reoptimize(module) + except ImportError as e: + if not first_build or 'ply' not in str(e): + raise + sys.stderr.write( + u"ERROR: cannot find pre-generated modules for the assumed 'ply' " + "version from above and/or cannot `import ply` to build generated " + "modules, aborting build; please either ensure that the source " + "archive containing the pre-generate modules is being used, or " + "that the python package 'ply' is installed and available for " + "import before attempting to use the setup.py to build this " + "package; please refer to the top level README for further " + "details\n" + ) + sys.exit(1) if __name__ == '__main__': # pragma: no cover - reoptimize_all(True) + reoptimize_all(True, '--build' in sys.argv) diff --git a/src/calmjs/parse/tests/test_parsers_optimize.py b/src/calmjs/parse/tests/test_parsers_optimize.py index c1f1c8b..3a2157c 100644 --- a/src/calmjs/parse/tests/test_parsers_optimize.py +++ b/src/calmjs/parse/tests/test_parsers_optimize.py @@ -5,12 +5,14 @@ import os import sys +from io import StringIO from shutil import rmtree from tempfile import mkdtemp from types import ModuleType from ply import lex from calmjs.parse.parsers import optimize from calmjs.parse.parsers import es5 +from calmjs.parse.utils import ply_dist class OptimizeTestCase(unittest.TestCase): @@ -22,16 +24,26 @@ def setUp(self): def tearDown(self): optimize.unlink = os.unlink optimize.import_module = importlib.import_module + optimize.ply_dist = ply_dist # undo whatever monkey patch that may have happened lex.open = open + def break_ply(self): + import ply + + def cleanup(): + sys.modules['ply'] = ply + + self.addCleanup(cleanup) + sys.modules['ply'] = None + def test_verify_paths(self): tempdir = mkdtemp() self.addCleanup(rmtree, tempdir) # create fake module files modules = [os.path.join(tempdir, name) for name in ( - 'foo.pyc', 'bar.pyc', 'foo.py')] + 'foo.pyc', 'bar.py', 'bar.pyc', 'foo.py')] for module in modules: with open(module, 'w'): @@ -42,11 +54,28 @@ def test_verify_paths(self): sorted(optimize.verify_paths(modules[:2])), ) - def test_unlink_modules(self): + def test_find_tab_paths(self): + fake_es5 = ModuleType('fake_es5') + fake_es5.lextab = 'some_lextab' + fake_es5.yacctab = 'some_yacctab' + paths, missing = optimize.find_tab_paths(fake_es5) + self.assertEqual([], paths) + self.assertEqual(['some_lextab', 'some_yacctab'], missing) + # ensure the parser exists es5.Parser() # should have created the optimized version of the file, if not # already exists + answers = [ + sys.modules[es5.lextab].__file__, + sys.modules[es5.yacctab].__file__, + ] + paths, missing = optimize.find_tab_paths(es5) + self.assertEqual(paths, answers) + self.assertEqual([], missing) + + def test_unlink_modules(self): + es5.Parser() p = sys.modules[es5.yacctab].__file__ self.assertTrue(os.path.exists(p)) # unlink has been patched out @@ -87,3 +116,166 @@ def test_reoptimize(self): def test_reoptimize_monkey_patched(self): optimize.reoptimize_all(True) self.assertIsNot(lex.open, open) + self.assertNotEqual(len(self.purged), 0) + + def test_optimize_build(self): + called = [] + + def sentinel(*a, **kw): + called.append(True) + + fake_es5 = ModuleType('fake_namespace.fake_es5') + fake_es5.Parser = sentinel + + # inject fake namespace and module + sys.modules['fake_namespace'] = ModuleType('fake_namespace') + self.addCleanup(sys.modules.pop, 'fake_namespace') + sys.modules['fake_namespace.fake_es5'] = fake_es5 + self.addCleanup(sys.modules.pop, 'fake_namespace.fake_es5') + + optimize.optimize_build('fake_namespace.fake_es5') + self.assertEqual(len(self.purged), 0) + self.assertTrue(called) + + def test_optimize_first_build(self): + optimize.reoptimize_all(True, first_build=True) + # shouldn't have purged any modules + self.assertEqual(len(self.purged), 0) + + def test_optimize_first_build_valid_with_broken_ply(self): + self.break_ply() + optimize.reoptimize_all(True, first_build=True) + # shouldn't have purged any modules + self.assertEqual(len(self.purged), 0) + + def test_assume_ply_version_default_ply(self): + # only applicable if no ply_dist found + optimize.ply_dist = None + stderr = sys.stderr + self.addCleanup(setattr, sys, 'stderr', stderr) + + # where ply is actually available; and since the real thing is + # expected to be present and usable, intersperse that real value + # into the expected string. + import ply + sys.stderr = StringIO() + optimize._assume_ply_version() + self.assertTrue(sys.stderr.getvalue().startswith( + "WARNING: cannot find distribution for 'ply'; using value " + "provided by ply, assuming 'ply==%s' for pre-generated modules" % ( + ply.__version__ + ))) + + def test_assume_ply_version_override_ply(self): + # can still override if ply is actually available + optimize.ply_dist = None + stderr = sys.stderr + self.addCleanup(setattr, sys, 'stderr', stderr) + + self.addCleanup(os.environ.pop, optimize._ASSUME_ENVVAR, None) + sys.stderr = StringIO() + os.environ[optimize._ASSUME_ENVVAR] = '0.9999' # should never exist + optimize._assume_ply_version() + self.assertTrue(sys.stderr.getvalue().startswith( + "WARNING: cannot find distribution for 'ply'; using environment " + "variable 'CALMJS_PARSE_ASSUME_PLY_VERSION', " + "assuming 'ply==0.9999' for pre-generated modules")) + + def test_assume_ply_version_no_ply(self): + # default when ply is fully broken. + optimize.ply_dist = None + stderr = sys.stderr + self.addCleanup(setattr, sys, 'stderr', stderr) + + self.break_ply() + sys.stderr = StringIO() + optimize._assume_ply_version() + self.assertTrue(sys.stderr.getvalue().startswith( + "WARNING: cannot find distribution for 'ply'; using default " + "value, assuming 'ply==3.11' for pre-generated modules")) + + def test_optimize_first_build_valid_with_broken_ply_error(self): + def fail_import(*a, **kw): + raise ImportError('no module named ply') + + optimize.import_module = fail_import + + with self.assertRaises(ImportError): + optimize.reoptimize_all() + + stderr = sys.stderr + + def cleanup(): + sys.stderr = stderr + + self.addCleanup(cleanup) + + sys.stderr = StringIO() + with self.assertRaises(SystemExit): + optimize.reoptimize_all(first_build=True) + + self.assertTrue(sys.stderr.getvalue().startswith( + "ERROR: cannot find pre-generated modules for the assumed 'ply' " + "version")) + + def test_optimize_first_build_assume_broken_ply_error(self): + optimize.ply_dist = None + + self.break_ply() + + def fail_import(*a, **kw): + raise ImportError('no module named ply') + + optimize.import_module = fail_import + + with self.assertRaises(ImportError): + optimize.reoptimize_all() + + stderr = sys.stderr + + def cleanup(): + sys.stderr = stderr + + self.addCleanup(cleanup) + + sys.stderr = StringIO() + with self.assertRaises(SystemExit): + optimize.reoptimize_all(first_build=True) + + lines = sys.stderr.getvalue().splitlines() + self.assertTrue(lines[0].startswith( + "WARNING: cannot find distribution for 'ply'; using default value" + )) + self.assertTrue(lines[1].startswith( + "ERROR: cannot find pre-generated modules for the assumed 'ply' " + "version")) + + def test_optimize_build_assume_broken_ply_but_available(self): + optimize.ply_dist = None + called = [] + + def sentinel(*a, **kw): + called.append(True) + + fake_es5 = ModuleType('fake_namespace.fake_es5') + fake_es5.Parser = sentinel + + # inject fake namespace and module + sys.modules['fake_namespace'] = ModuleType('fake_namespace') + self.addCleanup(sys.modules.pop, 'fake_namespace') + sys.modules['fake_namespace.fake_es5'] = fake_es5 + self.addCleanup(sys.modules.pop, 'fake_namespace.fake_es5') + stderr = sys.stderr + self.addCleanup(setattr, sys, 'stderr', stderr) + sys.stderr = StringIO() + + optimize.optimize_build('fake_namespace.fake_es5') + self.assertEqual(len(self.purged), 0) + self.assertTrue(called) + # this parser will not actually error as it does nothing; and + # so not actually care whether ply actually available here or + # not. + self.assertTrue(sys.stderr.getvalue().startswith( + "WARNING: cannot find distribution for 'ply'; " + )) + self.assertNotIn('ERROR', sys.stderr.getvalue()) diff --git a/src/calmjs/parse/utils.py b/src/calmjs/parse/utils.py index 3dde4de..51d82c9 100644 --- a/src/calmjs/parse/utils.py +++ b/src/calmjs/parse/utils.py @@ -13,6 +13,14 @@ from pkg_resources import working_set from pkg_resources import Requirement ply_dist = working_set.find(Requirement.parse('ply')) + # note that for **extremely** ancient versions of setuptools, e.g. + # setuptools<0.6c11, or some very non-standard environment that does + # not include the required metadata (e.g. pyinstaller without the + # required metadata), will require the following workaround... + if ply_dist is None: # pragma: no cover + from pkg_resources import Distribution + import ply + ply_dist = Distribution(project_name='ply', version=ply.__version__) except ImportError: # pragma: no cover ply_dist = None @@ -34,7 +42,7 @@ def repr_compat(s): return repr(s) -def generate_tab_names(name): +def generate_tab_names(name, _version='unknown'): """ Return the names to lextab and yacctab modules for the given module name. Typical usage should be like so:: @@ -44,8 +52,8 @@ def generate_tab_names(name): package_name, module_name = name.rsplit('.', 1) - version = ply_dist.version.replace( - '.', '_') if ply_dist is not None else 'unknown' + version = (ply_dist.version if ply_dist is not None else _version).replace( + '.', '_') data = (package_name, module_name, py_major, version) lextab = '%s.lextab_%s_py%d_ply%s' % data yacctab = '%s.yacctab_%s_py%d_ply%s' % data