diff --git a/.gitignore b/.gitignore index 4bf6500..9ec9301 100644 --- a/.gitignore +++ b/.gitignore @@ -108,3 +108,6 @@ ENV/ # VSCode .vscode/ test-reports/ + +# Gitlab +vars.env diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..f5197ad --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,52 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Python.gitlab-ci.yml +include: + - project: nci-gdc/gitlab-templates + ref: master + file: + - templates/global/full.yaml + - templates/python/full.yaml + +variables: + BASE_CONTAINER_VERSION: "2.0.1" + DOCKER_BUILDKIT: 1 + PIP_EXTRA_INDEX_URL: https://nexus.osdc.io/repository/pypi-all/simple + DOCKER_BUILD_OPTS: "--build-arg PIP_EXTRA_INDEX_URL=https://nexus.osdc.io/repository/pypi-all/simple" + +tox: + image: ${BASE_CONTAINER_REGISTRY}/${REPO_PY_VERSION}-builder:${BASE_CONTAINER_VERSION} + before_script: + - mkdir -p /usr/share/man/man1 + +release: + image: ${BASE_CONTAINER_REGISTRY}/${REPO_PY_VERSION}-builder:${BASE_CONTAINER_VERSION} + +.python_versions: + parallel: + matrix: + - REPO_PY_VERSION: [python3.8] + +docker_build: + rules: + - when: always + - if: '$CI_PIPELINE_SOURCE == "schedule"' + when: never + - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH + variables: + RELEASE_REGISTRY: containers.osdc.io/ncigdc # Override globally-defined RELEASE_REGISTRY + - if: $CI_COMMIT_REF_NAME =~ /main/ + variables: + RELEASE_REGISTRY: containers.osdc.io/ncigdc + script: + - echo "docker build ${DOCKER_BUILD_OPTS} -t ${RELEASE_REGISTRY}/$CI_PROJECT_NAME:$VERSION ." + - echo "docker push ${DOCKER_PUSH_OPTS} ${RELEASE_REGISTRY}/$CI_PROJECT_NAME:$VERSION" + - docker build . + --file ./Dockerfile + ${DOCKER_BUILD_OPTS} + --build-arg http_proxy + --build-arg https_proxy + --build-arg BASE_CONTAINER_VERSION + -t ${RELEASE_REGISTRY}/${CI_PROJECT_NAME}:${VERSION} + - docker push ${DOCKER_PUSH_OPTS} ${RELEASE_REGISTRY}/${CI_PROJECT_NAME}:${VERSION} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5b6065d..51c23bc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,13 +3,13 @@ default_language_version: python_venv: python3.8 repos: - - repo: git@github.com:Yelp/detect-secrets - rev: v1.0.3 + - repo: https://github.com/Yelp/detect-secrets + rev: v1.2.0 hooks: - id: detect-secrets args: ['--baseline', '.secrets.baseline'] - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.4.0 + rev: v4.2.0 hooks: - id: check-yaml - id: check-toml @@ -17,18 +17,18 @@ repos: - id: detect-aws-credentials args: ["--allow-missing-credentials"] - id: detect-private-key + - id: end-of-file-fixer - repo: https://github.com/pycqa/isort - rev: 5.8.0 + rev: 5.10.1 hooks: - id: isort name: isort - repo: https://github.com/psf/black - rev: 20.8b1 + rev: 22.3.0 hooks: - id: black - repo: https://github.com/pycqa/flake8 - rev: '3.9.2' + rev: '4.0.1' hooks: - id: flake8 additional_dependencies: [flake8-docstrings] - diff --git a/Dockerfile b/Dockerfile index 10e3ecd..52b0f9e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,26 +1,24 @@ -FROM quay.io/ncigdc/python38-builder as builder +ARG REGISTRY=quay.io +ARG BASE_CONTAINER_VERSION=2.0.1 + +FROM ${REGISTRY}/ncigdc/python3.8-builder:${BASE_CONTAINER_VERSION} as builder COPY ./ /opt WORKDIR /opt -RUN python -m pip install tox && tox - -# tox step builds sdist +RUN pip install tox && tox -e build -FROM quay.io/ncigdc/python38 +FROM ${REGISTRY}/ncigdc/python3.8:${BASE_CONTAINER_VERSION} COPY --from=builder /opt/dist/*.tar.gz /opt -COPY ./requirements.txt /opt/requirements.txt - -ENV BINARY=maflib +COPY requirements.txt /opt WORKDIR /opt -# Install package from sdist -RUN pip install -r requirements.txt \ - && pip install *.tar.gz \ - && rm -rf *.tar.gz requirements.txt +RUN pip install --no-deps -r requirements.txt \ + && pip install --no-deps *.tar.gz \ + && rm -f *.tar.gz requirements.txt ENTRYPOINT ["maflib"] diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index 62e0b85..0000000 --- a/Jenkinsfile +++ /dev/null @@ -1,6 +0,0 @@ -#!groovy - -library identifier: "jenkins-lib@master" -bioScriptedLibPipeline { - testRunner = 'tox' -} diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 80ac224..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,4 +0,0 @@ -# setuptools_scm includes most files -exclude .pre-commit-config.yaml -exclude .secrets.baseline -exclude .gitignore diff --git a/Makefile b/Makefile index ab166aa..017fae8 100644 --- a/Makefile +++ b/Makefile @@ -1,40 +1,47 @@ REPO = maf-lib + MODULE = maflib -GIT_SHORT_HASH:=$(shell git rev-parse --short HEAD) -GIT_COMMIT_HASH:=$(shell git rev-parse HEAD) -GIT_DESCRIBE:=$(shell git describe --tags) +# Redirect error when run in container +COMMIT_HASH:=$(shell git rev-parse HEAD 2>/dev/null) +GIT_DESCRIBE:=$(shell git describe --tags 2>/dev/null) DOCKER_REPO := quay.io/ncigdc -DOCKER_IMAGE_COMMIT := ${DOCKER_REPO}/${REPO}:${GIT_COMMIT_HASH} -DOCKER_IMAGE_LATEST := ${DOCKER_REPO}/${REPO}:latest +DOCKER_IMAGE_COMMIT := ${DOCKER_REPO}/${REPO}:${COMMIT_HASH} DOCKER_IMAGE_DESCRIBE := ${DOCKER_REPO}/${REPO}:${GIT_DESCRIBE} +DOCKER_IMAGE_LATEST := ${DOCKER_REPO}/${REPO}:latest + +# Env args +PIP_EXTRA_INDEX_URL ?= +http_proxy ?= +https_proxy ?= +REGISTRY ?= quay.io -TWINE_REPOSITORY_URL?="" .PHONY: version version-* version: - @python setup.py --version + @tox -e version version-docker: - @echo ${DOCKER_IMAGE_COMMIT} - -version-docker-tag: - @echo + @echo ${DOCKER_IMAGE_DESCRIBE} .PHONY: docker-login docker-login: docker login -u="${QUAY_USERNAME}" -p="${QUAY_PASSWORD}" quay.io +.PHONY: venv +venv: + @echo + rm -rf .venv/ + tox -r -e dev --devenv .venv .PHONY: init init-* init: init-pip init-hooks - -init-pip: +init-pip: init-venv @echo @echo -- Installing pip packages -- pip-sync requirements.txt dev-requirements.txt - python setup.py develop + python -m pip install -e . init-hooks: @echo @@ -47,69 +54,66 @@ init-venv: .PHONY: clean clean-* clean: clean-dirs - clean-dirs: rm -rf ./build/ rm -rf ./dist/ rm -rf ./*.egg-info/ - rm -rf ./.tox/ - rm -rf ./htmlcov - -clean-docker: - @docker rmi -f ${DOCKER_IMAGE_COMMIT} + rm -rf ./test-reports/ + rm -rf ./htmlcov/ + rm -rf coverage.xml + rm -rf .coverage .PHONY: requirements requirements-* requirements: init-venv requirements-prod requirements-dev - -requirements-prod: - pip-compile -o requirements.txt - requirements-dev: pip-compile -o dev-requirements.txt dev-requirements.in +requirements-prod: + pip-compile -o requirements.txt pyproject.toml + .PHONY: build build-* build: build-docker build-docker: clean + @echo @echo -- Building docker -- - docker build . -f Dockerfile \ - --build-arg http_proxy=${PROXY} \ - --build-arg https_proxy=${PROXY} \ + docker build . \ + --file ./Dockerfile \ + --build-arg http_proxy \ + --build-arg https_proxy \ + --build-arg REGISTRY \ -t "${DOCKER_IMAGE_COMMIT}" \ - -t "${DOCKER_IMAGE_LATEST}" + -t "${DOCKER_IMAGE_DESCRIBE}" \ + -t "${REPO}" build-pypi: clean - @tox -e check_dist + @echo + tox -e build .PHONY: lint test test-* tox test: tox +lint: + @echo + @echo -- Lint -- + tox -p -e flake8 test-unit: - @echo - @echo -- Unit Test -- - tox -e py36 + tox -e py3 test-docker: @echo tox: - @echo Running tox - tox -p all - -lint: @echo - @echo -- Lint -- - tox -p -e flake8 + TOX_PARALLEL_NO_SPINNER=1 tox -p --recreate .PHONY: publish-* publish-docker: - docker tag ${DOCKER_IMAGE_LATEST} ${DOCKER_IMAGE_DESCRIBE} docker push ${DOCKER_IMAGE_COMMIT} docker push ${DOCKER_IMAGE_DESCRIBE} publish-pypi: @echo - @echo Publishing dists - @python3 -m pip install --user --upgrade twine + @echo Publishing wheel python3 -m twine upload dist/* diff --git a/dev-requirements.in b/dev-requirements.in index f30c805..25efff2 100644 --- a/dev-requirements.in +++ b/dev-requirements.in @@ -1,5 +1,5 @@ -c requirements.txt -detect-secrets==1.0.3 +detect-secrets==1.2.0 isort flake8 flake8-docstrings diff --git a/dev-requirements.txt b/dev-requirements.txt index 7376909..32e950c 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -4,6 +4,8 @@ # # pip-compile --output-file=dev-requirements.txt dev-requirements.in # +--extra-index-url https://nexus.osdc.io/repository/pypi-all/simple + appdirs==1.4.4 # via virtualenv attrs==20.2.0 @@ -16,7 +18,7 @@ chardet==4.0.0 # via requests coverage==5.3 # via pytest-cov -detect-secrets==1.0.3 +detect-secrets==1.2.0 # via -r dev-requirements.in distlib==0.3.1 # via virtualenv diff --git a/maflib/column_types.py b/maflib/column_types.py index 4e0dc3e..46cc66f 100644 --- a/maflib/column_types.py +++ b/maflib/column_types.py @@ -101,7 +101,7 @@ def __validate__(self) -> Optional[str]: class StringColumn(NullableStringColumn): - """ A column where the value must be a non-empty string""" + """A column where the value must be a non-empty string""" EmptyStringMessage = "Empty string is not allowed" diff --git a/maflib/record.py b/maflib/record.py index 72f2a32..65c8a99 100644 --- a/maflib/record.py +++ b/maflib/record.py @@ -256,7 +256,7 @@ def value(self, key: TKey) -> Any: return None def column_values(self) -> List[Optional[Any]]: - """Gets the values for all columns in order. """ + """Gets the values for all columns in order.""" return [ (column.value if column is not None else None) for column in self.values() ] diff --git a/maflib/validation.py b/maflib/validation.py index fee18b1..6ce0691 100644 --- a/maflib/validation.py +++ b/maflib/validation.py @@ -82,7 +82,7 @@ def __str__(self) -> str: class MafValidationError: - """Stores a specific validation error and type """ + """Stores a specific validation error and type""" __IgnoringMessageFormat = "Ignoring MAF validation error: %s" diff --git a/pyproject.toml b/pyproject.toml index 34deae0..8886152 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,56 @@ [build-system] -requires = ["setuptools>=45", "wheel", "setuptools_scm[toml]>=6.2"] -build-backend = "setuptools.build_meta" +requires = ["flit_scm"] +build-backend = "flit_scm:buildapi" + +[project] +name = "bioinf-maflib" +description = "Library for creating and reading MAF files" +authors = [ + {name = "Charles Czysz", email = "czysz@uchicago.edu"} +] + +readme = "README.md" +requires-python = ">=3.8" +license = {file = "LICENSE"} +classifiers = [ + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only" +] +dynamic = ["version"] +dependencies = [] + +[project.urls] +homepage = "https://github.com/NCI-GDC/maf-lib" + +[project.scripts] +maflib = "maflib.__main__:main" [tool.setuptools_scm] -local_scheme = "dirty-tag" -version_scheme = "release-branch-semver" write_to = "maflib/_version.py" +local_scheme = "dirty-tag" +version_scheme = "python-simplified-semver" fallback_version = "0" +[tool.coverage.run] +source = ["maflib"] +branch = true +parallel = true + +[tool.flit.module] +name = "maflib" + +[tool.flit.sdist] +include = ["maflib/_version.py"] +exclude = [ + ".*", + "dev-requirements.*", + "Dockerfile", + "Jenkinsfile", + "*travis.yml", + "tox.ini", +] + [tool.black] line-length = 88 skip-string-normalization = true @@ -25,3 +68,6 @@ python_version = 3.8 disallow_untyped_defs = true warn_return_any = true warn_unused_configs = true + +[tool.coverage.report] +show_missing = true diff --git a/requirements.txt b/requirements.txt index 271db3b..3736d0b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,5 +2,6 @@ # This file is autogenerated by pip-compile with python 3.8 # To update, run: # -# pip-compile --output-file=requirements.txt +# pip-compile --output-file=requirements.txt pyproject.toml # +--extra-index-url https://nexus.osdc.io/repository/pypi-all/simple diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index fdbe74f..0000000 --- a/setup.cfg +++ /dev/null @@ -1,25 +0,0 @@ -[metadata] -name = bioinf_maflib -description = Maflib -url = https://github.com/NCI-GDC/maf-lib -author = Charles Czysz -author_email = czysz@uchicago.edu -python_requires = >=3.8 -license_files = - LICENSE -long_description = file:README.md -long_description_content_type = text/markdown -classifiers = - Programming Language :: Python :: 3 - Programming Language :: Python :: 3 :: Only - -[options] -install_requires = - -zip_safe = False -include_package_data = True -packages = find: - -[options.entry_points] -console_scripts = - maflib = maflib.__main__:main diff --git a/setup.py b/setup.py deleted file mode 100644 index f205a47..0000000 --- a/setup.py +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env python - -from setuptools import setup - -setup( - use_scm_version=True, - setup_requires=['setuptools_scm'], -) - -# __END__ diff --git a/tox.ini b/tox.ini index 8cd249b..6447fbf 100644 --- a/tox.ini +++ b/tox.ini @@ -1,58 +1,118 @@ [tox] -envlist = py38,flake8,check_dist,type -isolated_build = True +envlist = py3,help,version,flake8,type,cov +isolated_build = true -[flake8] -ignore = C901,D100,D104,E501,E302,E502,E126,E731,F841,W503,W605 -extend_ignore = D tests/ -exclude = - .tox, - .git, - .direnv, - __pycache__, - build, - dist, - *.pyc, - *.egg-info, - .eggs -docstring-convention=google +[tox:.package] +basepython = python3 [testenv] +setenv = + package = maflib + PYTHONHASHSEED = 0 +passenv = PIP_EXTRA_INDEX_URL +skip_install = + false +install_command = python -m pip install {opts} {packages} +deps = + -rdev-requirements.txt + -rrequirements.txt +commands = pytest -vv {posargs} + +[testenv:dev] +basepython = python3.8 + +[testenv:requirements] +deps = pip-tools +skip_install = + true +commands = + pip-compile -o dev-requirements.txt dev-requirements.in + +[testenv:help] +skip_install = + false deps = -rdev-requirements.txt -rrequirements.txt commands = - pytest -lvv \ - --cov-report=term-missing \ - --cov-report=html \ - --junitxml=test-reports/results.xml \ - --cov=maflib \ - tests/ + python -m {env:package} --help + +[testenv:cov] +skip_install=True +deps = + -rdev-requirements.txt + -rrequirements.txt +commands = + coverage run -m pytest tests/ + coverage combine + coverage report -m + coverage xml + coverage xml -o test-reports/results.xml [testenv:flake8] -skip_install = true +skip_install=True +deps = + -rdev-requirements.txt + -rrequirements.txt +commands = + flake8 {env:package}/ tests/ + +[testenv:type] +skip_install = false deps= - flake8 + -rrequirements.txt + -rdev-requirements.txt commands = - flake8 maflib/ tests/ setup.py + python -m mypy {env:package}/ + +[testenv:build] +skip_install= true +isolated_build = false +install_command = python -m pip install {opts} {packages} +deps= + flit + setuptools_scm +commands = + python -m setuptools_scm + python -m flit build -[testenv:check_dist] -skip_install = true +[testenv:publish] +isolated_build=False +skip_install=true +passenv = + TWINE_USERNAME + TWINE_PASSWORD + TWINE_REPOSITORY_URL +install_command = python -m pip install {opts} {packages} deps= - setuptools_scm - wheel + setuptools_scm + flit twine -whitelist_externals = rm commands = - python setup.py -q clean --all - python setup.py -q egg_info - python setup.py -q sdist --formats gztar bdist_wheel - twine check dist/* + python -m setuptools_scm + python -m flit build + twine check dist/* + twine upload dist/* -[testenv:type] -skip_install = true -deps= - mypy -commands = - python -m mypy maflib/ +[testenv:version] +skip_install=True +allowlist_externals = + git +deps = +commands = + git describe --tags +[flake8] +ignore = C901,D100,D104,E501,E302,E502,E126,E731,F841,W503,W605 +extend_ignore = D tests/ +exclude = + .tox, + .git, + .direnv, + __pycache__, + build, + dist, + *.pyc, + *.egg-info, + .eggs +docstring-convention=google