diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d874c8d --- /dev/null +++ b/.gitignore @@ -0,0 +1,214 @@ +### JetBrains template +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +### Python template +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +.idea/ \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..b653486 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 Oleg Taraasov + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/conda-export.py b/conda-export.py new file mode 100644 index 0000000..18f0bfa --- /dev/null +++ b/conda-export.py @@ -0,0 +1,108 @@ +import argparse +import sys +from pathlib import Path +from typing import List + +from conda.base.context import locate_prefix_by_name +from conda.cli.main import init_loggers +from conda.common.serialize import yaml_safe_dump +from conda_env.env import from_environment +from pip._vendor.pkg_resources import DistInfoDistribution +from pip._internal.utils.misc import get_installed_distributions + +__version__ = "0.0.1" + + +def find_site_packages(prefix: str) -> str: + """A naive hacky way to find site_packages directory for use with `pip`.""" + + lib_dir = Path(prefix).joinpath("lib") + if not lib_dir.exists(): + raise Exception(f"{lib_dir} does not exist!") + + site_packages = [ + str(path) + for path in map( + lambda x: x.joinpath("site-packages"), + lib_dir.glob("python*.*"), + ) + if path.exists() + ] + if len(site_packages) != 1: + raise Exception(f"Could not reliably find site-packages!") + + return site_packages[0] + + +def get_not_required( + packages: List[DistInfoDistribution], +) -> List[DistInfoDistribution]: + """Filter pip packages so that only leaf packages are left.""" + + dep_keys = set() + for dist in packages: + dep_keys.update(requirement.key for requirement in dist.requires()) + + return list({pkg for pkg in packages if pkg.key not in dep_keys}) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "-n", "--name", default=None, required=True, help="Conda environment name" + ) + parser.add_argument("-f", "--file", default=None, help="Output file name") + parser.add_argument( + "-V", "--version", action="version", version="%(prog)s " + __version__ + ) + + args = parser.parse_args() + + init_loggers() + + prefix = locate_prefix_by_name(args.name) + site_packages = find_site_packages(prefix) + + # All the packages in the environment: conda and pip (with versions) + env_all = from_environment(args.name, prefix, no_builds=True) + + # Conda packages that were explicitly installed (but not pip packages). + env_hist = from_environment(args.name, prefix, no_builds=True, from_history=True) + + # Get packages that no one depends on (leaves) from pip. + pip_leaves = { + dist.key + for dist in get_not_required(get_installed_distributions(paths=[site_packages])) + } + + # Strip version info from full conda packages. + conda_packages = { + pkg.split("=")[0] for pkg in env_all.dependencies.get("conda", []) + } + + # Leave just those pip packages that were not installed through conda. + pip_leaves = pip_leaves.difference(conda_packages) + + # Additionaly filter pip packages with conda's version of things. + if "pip" in env_all.dependencies: + conda_pip = {pkg.split("==")[0] for pkg in env_all.dependencies["pip"]} + pip_leaves = pip_leaves.intersection(conda_pip) + + final_dict = env_hist.to_dict() + final_dict["channels"] = env_all.channels + del final_dict["prefix"] + + if len(pip_leaves) > 0: + final_dict["dependencies"].append({"pip": list(pip_leaves)}) + + result = yaml_safe_dump(final_dict) + + if args.file is None: + print(result) + else: + with open(args.file, "w") as file: + file.write(result) + + +if __name__ == "__main__": + main() diff --git a/meta.yaml b/meta.yaml new file mode 100644 index 0000000..0b09515 --- /dev/null +++ b/meta.yaml @@ -0,0 +1,52 @@ +{% set name = "conda-export" %} +{% set version = "0.0.1" %} + +package: + name: {{ name|lower }} + version: {{ version }} + +source: + url: https://github.com/olegtarasov/{{ name }}/archive/v{{ version }}.tar.gz + #sha256: 2b3a0c466fb4a1014ea131c2b8ea7c519f9278eba73d6fcb361b7bdb4fd494e9 + # sha256 is the preferred checksum -- you can get it for a file with: + # `openssl sha256 `. + # You may need the openssl package, available on conda-forge: + # `conda install openssl -c conda-forge`` + +build: + noarch: python + number: 0 + script: "{{ PYTHON }} -m pip install . -vv" + entry_points: + - conda-export = conda_export:main + +requirements: + host: + - python >=3.6 + - pip + - setuptools + run: + - python >=3.6 + - conda + +test: + commands: + - conda-export -h + +about: + home: https://github.com/olegtarasov/conda-export + license: MIT + license_family: MIT + license_file: LICENSE + summary: 'Platform agnostic conda environment export' + description: | + An alternative to `conda env export` that helps create portable environment + specifications with minimal number of packages. Resulting specification is + similar to `conda env export --from-history`, but also includes packages + that were installed using `pip`. + doc_url: https://github.com/olegtarasov/conda-export + dev_url: https://github.com/olegtarasov/conda-export + +extra: + recipe-maintainers: + - olegtarasov diff --git a/readme.md b/readme.md new file mode 100644 index 0000000..57c8b02 --- /dev/null +++ b/readme.md @@ -0,0 +1,50 @@ +# conda-export +An alternative to `conda env export` that helps create portable environment +specifications with minimal number of packages. + +Resulting specification is similar to `conda env export --from-history`, but also +includes packages that were installed using `pip`. + +## Installation + +It makes sense to install `conda-export` into `base` environment and call it from +there, since it would be weird to install `conda` into your actual working env. + +```shell +conda install conda-export -n base -c conda-forge +``` + +## Usage + +```shell +conda-export -n [env name] -f [optional output file] +``` + +If `-f` is not specified, dumps the spec to the console. + +## Rationale +There are several options when you want to share conda environment specification: + +1. You can use `conda env export -n [name] -f [file]`. This command will give you a + full yaml specification with build versions, which is ideal for reproducibility on + **the same machine**. Unfortunately, this specification will most likely fail on a + different machine or different OS. +2. `conda env export --no-builds` is a little better, but it still contains specific + versions for all the packages, and such specification can still fail on a different + OS. You can postprocess the spec with a simple regex, removing version info, but + the spec will still contain all the packages that are installed in the environment. + Such a spec proves hard to maintain and reason about, and can still fail on + different OS. +3. Finally, you can use `conda env export --from-history`, which will give you only + those packages that you explicitly installed with `conda`. Versions will be + included only if you explicitly requested them upon package installation. This + would be the ideal solution, but unfortunately this command will not include + packages that were installed with `pip`. + +To circumvent all the above restrictions, I've created `conda-export` which generates +a spec with `--from-history` and adds `pip` packages, trying to minimize the number of +packages by including only leaves that no other packages depend on. + +## OMG, it uses private pip API! + +Yes, it does. Shame on me. \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..dc290a3 --- /dev/null +++ b/setup.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python + +from setuptools import setup +import shutil + + +# we need to rename the script because it's not a valid module name +shutil.copyfile("conda-export.py", "conda_export.py") + +setup( + name="conda-export", + version="0.0.1", + description="Platform agnostic conda environment export", + author="Oleg Tarasov", + url="https://github.com/olegtarasov/conda-export", + py_modules=["conda_export"], + entry_points={"console_scripts": ["conda-export=conda_export:main"]}, +)