Enhancements to papr tests

The PR contains several enhancements to our CI testing. - enable lint testing on Fedora - add Centos Atomic as test platform - integration tests on run on the OS natively (uncontainerized) - builds are done in containers - inclusion of Vagrant file for local testing Signed-off-by: baude <[email protected]> Closes: containers#18 Approved by: mheon
loongson-zjl · Nov 17, 2017 · d2eda1a · d2eda1a
1 parent bf8b9a3
commit d2eda1a
Show file tree

Hide file tree

Showing 38 changed files with 441 additions and 338 deletions.
diff --git a/.papr.sh b/.papr.sh
@@ -1,37 +1,101 @@
 #!/bin/bash
 set -xeuo pipefail
 
-export GOPATH=$HOME/gopath
+DIST=$(cat /etc/redhat-release  | awk '{print $1}')
+IMAGE=registry.fedoraproject.org/fedora:26
+PACKAGER=dnf
+if [[ ${DIST} != "Fedora" ]]; then
+    PACKAGER=yum
+    IMAGE=registry.centos.org/centos/centos:7
+fi
+
+if test -z "${INSIDE_CONTAINER:-}"; then
+    if [ -f /run/ostree-booted ]; then
+
+        # by default, the root LV on AH is only 3G, but we need a
+        # bit more for our tests
+        lvresize -r -L +4G atomicos/root
+
+        if [ ! -e /var/tmp/ostree-unlock-ovl.* ]; then
+            ostree admin unlock
+        fi
+    fi
+    # Restarting docker helps with permissions related to above.
+    systemctl restart docker
+
+    # somewhat mimic the spec conditional
+    source /etc/os-release
+    if [ "$ID" == fedora ]; then
+      PYTHON=python3
+    else
+      PYTHON=python
+    fi
+    docker run --rm \
+               --privileged \
+               -v $PWD:/go/src/github.com/projectatomic/libpod \
+               -v /etc/yum.repos.d:/etc/yum.repos.d.host:ro \
+               -v /:/host \
+               --workdir /go/src/github.com/projectatomic/libpod \
+               -e INSIDE_CONTAINER=1 \
+               -e PYTHON=$PYTHON \
+               ${IMAGE} /go/src/github.com/projectatomic/libpod/.papr.sh
+    systemd-detect-virt
+    ./test/test_runner.sh
+    exit 0
+fi
+
+export GOPATH=/go
 export PATH=$HOME/gopath/bin:$PATH
-export GOSRC=$HOME/gopath/src/github.com/projectatomic/libpod
-
-(mkdir -p $GOSRC && cd /code && cp -r . $GOSRC)
-
-dnf install -y \
-  bats \
-  btrfs-progs-devel \
-  bzip2 \
-  device-mapper-devel \
-  findutils \
-  git \
-  glib2-devel \
-  gnupg \
-  golang \
-  gpgme-devel \
-  libassuan-devel \
-  libseccomp-devel \
-  libselinux-devel \
-  skopeo-containers \
-  runc \
-  make \
-  ostree-devel \
-  python \
-  which
+export GOSRC=/$GOPATH/src/github.com/projectatomic/libpod
+
+${PACKAGER} install -y \
+              bats \
+              btrfs-progs-devel \
+              bzip2 \
+              device-mapper-devel \
+              findutils \
+              git \
+              glib2-devel \
+              gnupg \
+              golang \
+              gpgme-devel \
+              libassuan-devel \
+              libseccomp-devel \
+              libselinux-devel \
+              skopeo-containers \
+              runc \
+              make \
+              ostree-devel \
+              python \
+              which\
+              golang-github-cpuguy83-go-md2man
+
 
 # PAPR adds a merge commit, for testing, which fails the
 # short-commit-subject validation test, so tell git-validate.sh to only check
 # up to, but not including, the merge commit.
 export GITVALIDATE_TIP=$(cd $GOSRC; git log -2 --pretty='%H' | tail -n 1)
 export TAGS="seccomp $($GOSRC/hack/btrfs_tag.sh) $($GOSRC/hack/libdm_tag.sh) $($GOSRC/hack/btrfs_installed_tag.sh) $($GOSRC/hack/ostree_tag.sh) $($GOSRC/hack/selinux_tag.sh)"
-make -C $GOSRC binaries install.tools all gofmt localintegration testunit TAGS="${TAGS}"
-#make -C $GOSRC lint
+
+make gofmt TAGS="${TAGS}"
+make testunit TAGS="${TAGS}"
+make install.tools TAGS="${TAGS}"
+
+# Only check lint and gitvalidation on more recent
+# distros with updated git and tooling
+if [[ ${PACKAGER} != "yum" ]]; then
+    HEAD=$GITVALIDATE_TIP make -C $GOSRC .gitvalidation TAGS="${TAGS}"
+    make lint
+    dnf install -y --installroot /host bats
+fi
+
+make TAGS="${TAGS}"
+make TAGS="${TAGS}" install PREFIX=/host/usr
+make TAGS="${TAGS}" test-binaries
+
+if [[ ${PACKAGER} == "yum" ]]; then
+    # Install EPEL to get Bats
+    ${PACKAGER} -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
+    ${PACKAGER} install --downloadonly --downloaddir=/tmp bats
+    cd /host && rpm2cpio /tmp/bats*.rpm | cpio -ivd
+fi
diff --git a/.papr.yml b/.papr.yml
@@ -13,8 +13,14 @@ required: true
 timeout: 45m
 
 tests:
-  # mount yum repos to inherit injected mirrors from PAPR
-  - docker run --net=host --privileged -v /etc/yum.repos.d:/etc/yum.repos.d.host:ro
-    -v /tmp:/tmp -v /var/tmp:/var/tmp
-    -v $PWD:/code registry.fedoraproject.org/fedora:26 sh -c
-    "cp -fv /etc/yum.repos.d{.host/*.repo,} && /code/.papr.sh"
+  - CRIO_ROOT=/var/tmp/checkout KPOD_BINARY=/usr/bin/kpod CONMON_BINARY=/usr/libexec/crio/conmon PAPR=1 sh .papr.sh
+
+---
+
+inherit: true
+host:
+  distro: centos/7/atomic/alpha
+  specs:
+     ram: 8192
+
+context: centos/7/atomic/alpha
diff --git a/.tool/lint b/.tool/lint
@@ -13,7 +13,9 @@ if [ ! -f ${LINTER} ]; then
 	exit 1
 fi
 
-PKGS=$(find . -type d -not -path . -a -not -iwholename '*.git*' -a -not -iname '.tool' -a -not -iwholename '*vendor*' -a -not -iname 'hack' -a -not -iwholename '*.artifacts*' -a -not -iwholename '*contrib*' -a -not -iwholename '*test*' -a -not -iwholename '*logo*' -a -not -iwholename '*conmon*' -a -not -iwholename '*completions*' -a -not -iwholename '*docs*' -a -not -iwholename '*pause*')
+PKGS=$(find . -type d -not -path . -a -not -iwholename '*.git*' -a -not -iname '.tool' -a -not -iwholename '*vendor*' -a -not -iname 'hack' -a -not -iwholename '*.artifacts*' -a -not -iwholename '*contrib*' -a -not -iwholename '*test*' -a -not -iwholename '*logo*' -a -not -iwholename '*conmon*' -a -not -iwholename '*completions*' -a -not -iwholename '*docs*' -a -not -iwholename '*pause*' -a -not -iwholename './_output*')
+
+echo $PKGS
 
 # Execute the linter
 ${LINTER} \
@@ -40,5 +42,6 @@ ${LINTER} \
 	--exclude='duplicate of.*_test.go.*\(dupl\)$'\
 	--exclude='cmd\/client\/.*\.go.*\(dupl\)$'\
 	--exclude='vendor\/.*'\
+	--exclude='kpod\/.*'\
 	--exclude='server\/seccomp\/.*\.go.*$'\
 	${PKGS[@]}
diff --git a/.travis.yml b/.travis.yml
@@ -25,6 +25,10 @@ before_script:
   - export PATH=$HOME/gopath/bin:$PATH
   - export LD_LIBRARY_PATH=/usr/local/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}
 
+env:
+    global:
+        - TRAVIS=1
+
 jobs:
   include:
     - stage: Build and Verify

diff --git a/Makefile b/Makefile
@@ -1,5 +1,6 @@
 GO ?= go
-EPOCH_TEST_COMMIT ?= 2b74391
+EPOCH_TEST_COMMIT ?= 5cfd7a3
+HEAD ?= HEAD
 PROJECT := github.com/projectatomic/libpod
 GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null)
 GIT_BRANCH_CLEAN := $(shell echo $(GIT_BRANCH) | sed -e "s/[^[:alnum:]]/-/g")
@@ -41,6 +42,8 @@ KPOD_LDFLAGS := -X main.kpodVersion=${KPOD_VERSION}
 LDFLAGS := -ldflags '${BASE_LDFLAGS}'
 LDFLAGS_KPOD := -ldflags '${BASE_LDFLAGS} ${KPOD_LDFLAGS}'
 
+BOX="fedora_atomic"
+
 all: binaries docs
 
 default: help
@@ -114,6 +117,9 @@ testunit:
 localintegration: test-binaries
 	bash -i ./test/test_runner.sh ${TESTFLAGS}
 
+vagrant-check:
+	BOX=$(BOX) sh ./vagrant.sh
+
 binaries: conmon kpod
 
 test-binaries: test/bin2img/bin2img test/copyimg/copyimg test/checkseccomp/checkseccomp
@@ -151,13 +157,8 @@ uninstall:
 	done
 
 .PHONY: .gitvalidation
-# When this is running in travis, it will only check the travis commit range
 .gitvalidation: .gopathok
-ifeq ($(TRAVIS),true)
-	GIT_CHECK_EXCLUDE="./vendor" $(GOPATH)/bin/git-validation -q -run DCO,short-subject,dangling-whitespace
-else
-	GIT_CHECK_EXCLUDE="./vendor" $(GOPATH)/bin/git-validation -v -run DCO,short-subject,dangling-whitespace -range $(EPOCH_TEST_COMMIT)..HEAD
-endif
+	GIT_CHECK_EXCLUDE="./vendor" $(GOPATH)/bin/git-validation -v -run DCO,short-subject,dangling-whitespace -range $(EPOCH_TEST_COMMIT)..$(HEAD)
 
 .PHONY: install.tools
 

diff --git a/Vagrantfile b/Vagrantfile
@@ -0,0 +1,35 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+# All Vagrant configuration is done below. The "2" in Vagrant.configure
+# configures the configuration version (we support older styles for
+# backwards compatibility). Please don't change it unless you know what
+# you're doing.
+Vagrant.configure(2) do |config|
+    config.vm.provider "libvirt" do |libvirt, override|
+        libvirt.memory = 3096
+        libvirt.cpus = 3
+	libvirt.storage :file,
+		:type => 'qcow2'
+    end
+    config.vm.synced_folder ".", "/home/vagrant/sync", disabled: true
+    config.vm.synced_folder ".", "/home/vagrant/libpod", type: "rsync", rsync__exclude: ["_output"]
+
+  # The most common configuration options are documented and commented below.
+  # For a complete reference, please see the online documentation at
+  # https://docs.vagrantup.com.
+
+  # Every Vagrant development environment requires a box. You can search for
+  # boxes at https://atlas.hashicorp.com/search.
+    config.vm.define "fedora_atomic" do |fedora_atomic|
+        fedora_atomic.vm.box = "fedora_atomic"
+        fedora_atomic.vm.box_url = "https://getfedora.org/atomic_vagrant_libvirt_latest"
+    end
+    config.vm.define "centos_atomic" do |centos_atomic|
+        centos_atomic.vm.box =  "centos_atomic"
+        centos_atomic.vm.box_url = "https://ci.centos.org/artifacts/sig-atomic/centos-continuous/images/cloud/latest/images/centos-atomic-host-7-vagrant-libvirt.box"
+    end
+    config.vm.define "fedora_cloud" do |fedora_cloud|
+        fedora_cloud.vm.box =  "fedora/26-cloud-base"
+    end
+end
diff --git a/libpod/container.go b/libpod/container.go
@@ -411,14 +411,17 @@ func (c *Container) Attach(noStdin bool, keys string, attached chan<- bool) erro
 	resize := make(chan remotecommand.TerminalSize)
 	defer close(resize)
 	err = c.attachContainerSocket(resize, noStdin, detachKeys, attached)
-	if err != nil {
-		return err
-	}
+
+	return err
+
 	// TODO
 	// Re-enable this when mheon is done wth it
+	//if err != nil {
+	//	return err
+	//}
 	//c.ContainerStateToDisk(c)
 
-	return nil
+	//return err
 }
 
 // Mount mounts a container's filesystem on the host

diff --git a/libpod/diff.go b/libpod/diff.go
@@ -44,7 +44,7 @@ func (r *Runtime) getLayerID(id string) (string, error) {
 	return toLayer, nil
 }
 
-func (r *Runtime) getLayerParent(layerID string) (string, error) {
+func (r *Runtime) getLayerParent(layerID string) (string, error) { //nolint
 	layer, err := r.store.Layer(layerID)
 	if err != nil {
 		return "", err

diff --git a/libpod/oci.go b/libpod/oci.go
@@ -256,18 +256,18 @@ func (r *OCIRuntime) createContainer(ctr *Container, cgroupParent string) error
 
 // updateContainerStatus retrieves the current status of the container from the
 // runtime
-func (r *OCIRuntime) updateContainerStatus(ctr *Container) error {
+// remove nolint when implemented
+func (r *OCIRuntime) updateContainerStatus(ctr *Container) error { //nolint
 	return ErrNotImplemented
 }
 
 // startContainer starts the given container
-func (r *OCIRuntime) startContainer(ctr *Container) error {
+// remove nolint when function is complete
+func (r *OCIRuntime) startContainer(ctr *Container) error { //nolint
 	// TODO: streams should probably *not* be our STDIN/OUT/ERR - redirect to buffers?
-	if err := utils.ExecCmdWithStdStreams(os.Stdin, os.Stdout, os.Stderr, r.path, "start", ctr.ID()); err != nil {
-		return err
-	}
+	err := utils.ExecCmdWithStdStreams(os.Stdin, os.Stdout, os.Stderr, r.path, "start", ctr.ID())
 
 	// TODO record start time in container struct
 
-	return nil
+	return err
 }
diff --git a/libpod/runtime_ctr.go b/libpod/runtime_ctr.go
@@ -223,6 +223,7 @@ func (r *Runtime) removeMultipleContainers(containers []storage.Container) error
 }
 
 // ContainerConfigToDisk saves a container's nonvolatile configuration to disk
-func (r *Runtime) containerConfigToDisk(ctr *Container) error {
+// remove nolint when implemented
+func (r *Runtime) containerConfigToDisk(ctr *Container) error { //nolint
 	return ErrNotImplemented
 }
diff --git a/libpod/runtime_img.go b/libpod/runtime_img.go
@@ -118,7 +118,7 @@ type Image struct {
 	Name           string
 	ID             string
 	fqname         string
-	hasImageLocal  bool
+	hasImageLocal  bool //nolint
 	runtime        *Runtime
 	Registry       string
 	ImageName      string
@@ -359,10 +359,7 @@ func (k *Image) HasLatest() (bool, error) {
 		return false, err
 	}
 	_, _, err = pullRef.(types.ImageSource).GetManifest()
-	if err != nil {
-		return false, err
-	}
-	return false, nil
+	return false, err
 }
 
 // Pull is a wrapper function to pull and image
@@ -1155,7 +1152,8 @@ func ParseImageNames(names []string) (tags, digests []string, err error) {
 	return tags, digests, nil
 }
 
-func annotations(manifest []byte, manifestType string) map[string]string {
+// Remove nolint when used
+func annotations(manifest []byte, manifestType string) map[string]string { //nolint
 	annotations := make(map[string]string)
 	switch manifestType {
 	case ociv1.MediaTypeImageManifest:

diff --git a/pkg/storage/image.go b/pkg/storage/image.go
@@ -525,7 +525,7 @@ func GetImageService(store storage.Store, defaultTransport string, insecureRegis
 	is := &imageService{
 		store:                 store,
 		defaultTransport:      defaultTransport,
-		indexConfigs:          make(map[string]*indexInfo, 0),
+		indexConfigs:          make(map[string]*indexInfo),
 		insecureRegistryCIDRs: make([]*net.IPNet, 0),
 		registries:            cleanRegistries,
 	}