Merge pull request #53 from NCI-GDC/feat/BINF-679-gitlab-ci

[BINF-670] Gitlab CI

.gitignore

@@ -108,3 +108,6 @@ ENV/
 # VSCode
 .vscode/
 test-reports/
+
+# Gitlab
+vars.env

.gitlab-ci.yml

@@ -0,0 +1,52 @@
+# To contribute improvements to CI/CD templates, please follow the Development guide at:
+# https://docs.gitlab.com/ee/development/cicd/templates.html
+# This specific template is located at:
+# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Python.gitlab-ci.yml
+include:
+  - project: nci-gdc/gitlab-templates
+    ref: master
+    file:
+      - templates/global/full.yaml
+      - templates/python/full.yaml
+
+variables:
+ BASE_CONTAINER_VERSION: "2.0.1"
+ DOCKER_BUILDKIT: 1
+ PIP_EXTRA_INDEX_URL: https://nexus.osdc.io/repository/pypi-all/simple
+ DOCKER_BUILD_OPTS: "--build-arg PIP_EXTRA_INDEX_URL=https://nexus.osdc.io/repository/pypi-all/simple"
+
+tox:
+  image: ${BASE_CONTAINER_REGISTRY}/${REPO_PY_VERSION}-builder:${BASE_CONTAINER_VERSION}
+  before_script:
+    - mkdir -p /usr/share/man/man1
+
+release:
+  image: ${BASE_CONTAINER_REGISTRY}/${REPO_PY_VERSION}-builder:${BASE_CONTAINER_VERSION}
+
+.python_versions:
+  parallel:
+    matrix:
+      - REPO_PY_VERSION: [python3.8]
+
+docker_build:
+  rules:
+    - when: always
+    - if: '$CI_PIPELINE_SOURCE == "schedule"'
+      when: never
+    - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH
+      variables:
+        RELEASE_REGISTRY: containers.osdc.io/ncigdc  # Override globally-defined RELEASE_REGISTRY
+    - if: $CI_COMMIT_REF_NAME =~ /main/
+      variables:
+        RELEASE_REGISTRY: containers.osdc.io/ncigdc
+  script:
+    - echo "docker build ${DOCKER_BUILD_OPTS} -t ${RELEASE_REGISTRY}/$CI_PROJECT_NAME:$VERSION ."
+    - echo "docker push ${DOCKER_PUSH_OPTS} ${RELEASE_REGISTRY}/$CI_PROJECT_NAME:$VERSION"
+    - docker build . 
+      --file ./Dockerfile
+      ${DOCKER_BUILD_OPTS}
+      --build-arg http_proxy
+      --build-arg https_proxy
+      --build-arg BASE_CONTAINER_VERSION
+      -t ${RELEASE_REGISTRY}/${CI_PROJECT_NAME}:${VERSION}
+    - docker push ${DOCKER_PUSH_OPTS} ${RELEASE_REGISTRY}/${CI_PROJECT_NAME}:${VERSION}

.pre-commit-config.yaml

@@ -3,32 +3,32 @@ default_language_version:
   python_venv: python3.8
 
 repos:
-  - repo: git@github.com:Yelp/detect-secrets
-    rev: v1.0.3
+  - repo: https://github.com/Yelp/detect-secrets
+    rev: v1.2.0
     hooks:
       - id: detect-secrets
         args: ['--baseline', '.secrets.baseline']
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.4.0
+    rev: v4.2.0
     hooks:
       - id: check-yaml
       - id: check-toml
       - id: check-json
       - id: detect-aws-credentials
         args: ["--allow-missing-credentials"]
       - id: detect-private-key
+      - id: end-of-file-fixer
   - repo: https://github.com/pycqa/isort
-    rev: 5.8.0
+    rev: 5.10.1
     hooks:
       - id: isort
         name: isort
   - repo: https://github.com/psf/black
-    rev: 20.8b1
+    rev: 22.3.0
     hooks:
       - id: black
   - repo: https://github.com/pycqa/flake8
-    rev: '3.9.2'
+    rev: '4.0.1'
     hooks:
     -   id: flake8
         additional_dependencies: [flake8-docstrings]
-

Dockerfile

@@ -1,26 +1,24 @@
-FROM quay.io/ncigdc/python38-builder as builder
+ARG REGISTRY=quay.io
+ARG BASE_CONTAINER_VERSION=2.0.1
+
+FROM ${REGISTRY}/ncigdc/python3.8-builder:${BASE_CONTAINER_VERSION} as builder
 
 COPY ./ /opt
 
 WORKDIR /opt
 
-RUN python -m pip install tox && tox
-
-# tox step builds sdist
+RUN pip install tox && tox -e build
 
-FROM quay.io/ncigdc/python38
+FROM ${REGISTRY}/ncigdc/python3.8:${BASE_CONTAINER_VERSION}
 
 COPY --from=builder /opt/dist/*.tar.gz /opt
-COPY ./requirements.txt /opt/requirements.txt
-
-ENV BINARY=maflib
+COPY requirements.txt /opt
 
 WORKDIR /opt
 
-# Install package from sdist
-RUN pip install -r requirements.txt \
-	&& pip install *.tar.gz \
-	&& rm -rf *.tar.gz requirements.txt
+RUN pip install --no-deps -r requirements.txt \
+	&& pip install --no-deps *.tar.gz \
+	&& rm -f *.tar.gz requirements.txt
 
 ENTRYPOINT ["maflib"]
 

Makefile

@@ -1,40 +1,47 @@
 REPO = maf-lib
+
 MODULE = maflib
 
-GIT_SHORT_HASH:=$(shell git rev-parse --short HEAD)
-GIT_COMMIT_HASH:=$(shell git rev-parse HEAD)
-GIT_DESCRIBE:=$(shell git describe --tags)
+# Redirect error when run in container
+COMMIT_HASH:=$(shell git rev-parse HEAD 2>/dev/null)
+GIT_DESCRIBE:=$(shell git describe --tags 2>/dev/null)
 
 DOCKER_REPO := quay.io/ncigdc
-DOCKER_IMAGE_COMMIT := ${DOCKER_REPO}/${REPO}:${GIT_COMMIT_HASH}
-DOCKER_IMAGE_LATEST := ${DOCKER_REPO}/${REPO}:latest
+DOCKER_IMAGE_COMMIT := ${DOCKER_REPO}/${REPO}:${COMMIT_HASH}
 DOCKER_IMAGE_DESCRIBE := ${DOCKER_REPO}/${REPO}:${GIT_DESCRIBE}
+DOCKER_IMAGE_LATEST := ${DOCKER_REPO}/${REPO}:latest
+
+# Env args
+PIP_EXTRA_INDEX_URL ?=
+http_proxy ?=
+https_proxy ?=
+REGISTRY ?= quay.io
 
-TWINE_REPOSITORY_URL?=""
 
 .PHONY: version version-*
 version:
-	@python setup.py --version
+	@tox -e version
 
 version-docker:
-	@echo ${DOCKER_IMAGE_COMMIT}
-
-version-docker-tag:
-	@echo
+	@echo ${DOCKER_IMAGE_DESCRIBE}
 
 .PHONY: docker-login
 docker-login:
 	docker login -u="${QUAY_USERNAME}" -p="${QUAY_PASSWORD}" quay.io
 
+.PHONY: venv
+venv:
+	@echo
+	rm -rf .venv/
+	tox -r -e dev --devenv .venv
 
 .PHONY: init init-*
 init: init-pip init-hooks
-
-init-pip:
+init-pip: init-venv
 	@echo
 	@echo -- Installing pip packages --
 	pip-sync requirements.txt dev-requirements.txt
-	python setup.py develop
+	python -m pip install -e .
 
 init-hooks:
 	@echo
@@ -47,69 +54,66 @@ init-venv:
 
 .PHONY: clean clean-*
 clean: clean-dirs
-
 clean-dirs:
 	rm -rf ./build/
 	rm -rf ./dist/
 	rm -rf ./*.egg-info/
-	rm -rf ./.tox/
-	rm -rf ./htmlcov
-
-clean-docker:
-	@docker rmi -f ${DOCKER_IMAGE_COMMIT}
+	rm -rf ./test-reports/
+	rm -rf ./htmlcov/
+	rm -rf coverage.xml
+	rm -rf .coverage
 
 .PHONY: requirements requirements-*
 requirements: init-venv requirements-prod requirements-dev
-
-requirements-prod:
-	pip-compile -o requirements.txt
-
 requirements-dev:
 	pip-compile -o dev-requirements.txt dev-requirements.in
 
+requirements-prod:
+	pip-compile -o requirements.txt pyproject.toml
+
 .PHONY: build build-*
 
 build: build-docker
 
 build-docker: clean
+	@echo
 	@echo -- Building docker --
-	docker build . -f Dockerfile \
-		--build-arg http_proxy=${PROXY} \
-		--build-arg https_proxy=${PROXY} \
+	docker build . \
+		--file ./Dockerfile \
+		--build-arg http_proxy \
+		--build-arg https_proxy \
+		--build-arg REGISTRY \
 		-t "${DOCKER_IMAGE_COMMIT}" \
-		-t "${DOCKER_IMAGE_LATEST}"
+		-t "${DOCKER_IMAGE_DESCRIBE}" \
+		-t "${REPO}"
 
 build-pypi: clean
-	@tox -e check_dist
+	@echo
+	tox -e build
 
 .PHONY: lint test test-* tox
 test: tox
+lint:
+	@echo
+	@echo -- Lint --
+	tox -p -e flake8
 
 test-unit:
-	@echo
-	@echo -- Unit Test --
-	tox -e py36
+	tox -e py3
 
 test-docker:
 	@echo
 
 tox:
-	@echo Running tox
-	tox -p all
-
-lint:
 	@echo
-	@echo -- Lint --
-	tox -p -e flake8
+	TOX_PARALLEL_NO_SPINNER=1 tox -p --recreate
 
 .PHONY: publish-*
 publish-docker:
-	docker tag ${DOCKER_IMAGE_LATEST} ${DOCKER_IMAGE_DESCRIBE}
 	docker push ${DOCKER_IMAGE_COMMIT}
 	docker push ${DOCKER_IMAGE_DESCRIBE}
 
 publish-pypi:
 	@echo
-	@echo Publishing dists
-	@python3 -m pip install --user --upgrade twine
+	@echo Publishing wheel
 	python3 -m twine upload dist/*

dev-requirements.in

@@ -1,5 +1,5 @@
 -c requirements.txt
-detect-secrets==1.0.3
+detect-secrets==1.2.0
 isort
 flake8
 flake8-docstrings

dev-requirements.txt

@@ -4,6 +4,8 @@
 #
 #    pip-compile --output-file=dev-requirements.txt dev-requirements.in
 #
+--extra-index-url https://nexus.osdc.io/repository/pypi-all/simple
+
 appdirs==1.4.4
     # via virtualenv
 attrs==20.2.0
@@ -16,7 +18,7 @@ chardet==4.0.0
     # via requests
 coverage==5.3
     # via pytest-cov
-detect-secrets==1.0.3
+detect-secrets==1.2.0
     # via -r dev-requirements.in
 distlib==0.3.1
     # via virtualenv

maflib/column_types.py

@@ -101,7 +101,7 @@ def __validate__(self) -> Optional[str]:
 
 
 class StringColumn(NullableStringColumn):
-    """ A column where the value must be a non-empty string"""
+    """A column where the value must be a non-empty string"""
 
     EmptyStringMessage = "Empty string is not allowed"
 

maflib/record.py

@@ -256,7 +256,7 @@ def value(self, key: TKey) -> Any:
             return None
 
     def column_values(self) -> List[Optional[Any]]:
-        """Gets the values for all columns in order. """
+        """Gets the values for all columns in order."""
         return [
             (column.value if column is not None else None) for column in self.values()
         ]

maflib/validation.py

@@ -82,7 +82,7 @@ def __str__(self) -> str:
 
 
 class MafValidationError:
-    """Stores a specific validation error and type """
+    """Stores a specific validation error and type"""
 
     __IgnoringMessageFormat = "Ignoring MAF validation error: %s"