From 16370b1faea57e61d5a90b079206a3c7dbbe2065 Mon Sep 17 00:00:00 2001 From: bjaroszewski Date: Thu, 18 Apr 2019 11:04:15 +0200 Subject: [PATCH] Cleanup before release --- .github/CODE_OF_CONDUCT.md | 77 +++++ .github/CONTRIBUTING.md | 20 ++ .github/ISSUE_TEMPLATE.md | 27 ++ .github/PULL_REQUEST_TEMPLATE.md | 20 ++ .gitignore | 3 + .travis.yml | 20 ++ ACKNOWLEDGEMENT.md | 5 + CHANGELOG.md | 22 +- CONTRIBUTING.md | 22 ++ Godeps/Godeps.json | 34 --- Godeps/Readme | 5 - Gopkg.lock | 274 ++++++++++++++++++ Gopkg.toml | 27 ++ LICENSE | 202 +++++++++++++ LICENSE.md | 7 - Makefile | 44 --- README.md | 255 ++++++++++++---- TODO.md | 71 ----- build/Dockerfile | 15 - build/build.sh | 24 -- build/configure.sh | 19 -- build/install-libgit2.sh | 26 -- build/install-libssh2.sh | 26 -- build/install-openssl.sh | 21 -- config/config.go | 46 ++- config/load.go | 39 ++- config/load_test.go | 28 +- config/mock/mock.go | 33 ++- config/test-fixtures/test_ssh_key | 27 ++ glide.lock | 28 -- glide.yaml | 16 - kv/api.go | 37 +++ kv/branch.go | 64 ++-- kv/branch_test.go | 69 +++++ kv/filehandler.go | 176 +++++++++++ kv/filehandler_test.go | 175 +++++++++++ kv/handler.go | 126 +++++++- kv/init.go | 109 ++++--- kv/kv.go | 79 +++-- kv/kv_test.go | 108 +++++++ kv/mocks/kv.go | 78 +++++ kv/mocks/repository.go | 109 +++++++ kv/path.go | 97 +++++++ kv/path_test.go | 175 +++++++++++ kv/ref.go | 39 ++- kv/ref_test.go | 78 +++++ kv/update.go | 75 ++++- kv/update_test.go | 89 ++++++ main.go | 23 +- repository/auth.go | 48 +++ repository/auth_test.go | 84 ++++++ repository/checkout.go | 130 ++++++--- repository/checkout_test.go | 61 ++-- repository/clone.go | 29 +- repository/clone_test.go | 47 +-- repository/diff.go | 87 +++--- repository/diff_test.go | 79 ++--- repository/load.go | 26 +- repository/load_test.go | 110 ++++--- repository/mock/mock.go | 39 --- repository/mocks/repository.go | 91 ++++++ repository/pull.go | 101 +++---- repository/pull_test.go | 58 ++-- repository/ref.go | 28 +- repository/ref_test.go | 68 +++-- repository/repository.go | 115 +++++--- repository/repository_test.go | 61 ++-- runner/runner.go | 44 ++- testutil/git_repo.go | 204 ------------- testutil/test-fixtures/example.json | 17 -- testutil/test-fixtures/example/foo | 1 - testutil/test-fixtures/example/tree/bang | 1 - testutil/test-fixtures/example/types/bar.json | 3 - version.go | 18 +- watcher/interval.go | 75 +++-- watcher/interval_test.go | 62 ++-- watcher/watcher.go | 30 +- watcher/webhook.go | 77 ++--- 78 files changed, 3626 insertions(+), 1357 deletions(-) create mode 100644 .github/CODE_OF_CONDUCT.md create mode 100644 .github/CONTRIBUTING.md create mode 100644 .github/ISSUE_TEMPLATE.md create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 .travis.yml create mode 100644 ACKNOWLEDGEMENT.md create mode 100644 CONTRIBUTING.md delete mode 100644 Godeps/Godeps.json delete mode 100644 Godeps/Readme create mode 100644 Gopkg.lock create mode 100644 Gopkg.toml create mode 100644 LICENSE delete mode 100644 LICENSE.md delete mode 100644 Makefile delete mode 100644 TODO.md delete mode 100644 build/Dockerfile delete mode 100755 build/build.sh delete mode 100755 build/configure.sh delete mode 100755 build/install-libgit2.sh delete mode 100755 build/install-libssh2.sh delete mode 100755 build/install-openssl.sh create mode 100644 config/test-fixtures/test_ssh_key delete mode 100644 glide.lock delete mode 100644 glide.yaml create mode 100644 kv/api.go create mode 100644 kv/branch_test.go create mode 100644 kv/filehandler.go create mode 100644 kv/filehandler_test.go create mode 100644 kv/kv_test.go create mode 100644 kv/mocks/kv.go create mode 100644 kv/mocks/repository.go create mode 100644 kv/path.go create mode 100644 kv/path_test.go create mode 100644 kv/ref_test.go create mode 100644 kv/update_test.go create mode 100644 repository/auth.go create mode 100644 repository/auth_test.go delete mode 100644 repository/mock/mock.go create mode 100644 repository/mocks/repository.go delete mode 100644 testutil/git_repo.go delete mode 100644 testutil/test-fixtures/example.json delete mode 100644 testutil/test-fixtures/example/foo delete mode 100644 testutil/test-fixtures/example/tree/bang delete mode 100644 testutil/test-fixtures/example/types/bar.json diff --git a/.github/CODE_OF_CONDUCT.md b/.github/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..e3cde9f --- /dev/null +++ b/.github/CODE_OF_CONDUCT.md @@ -0,0 +1,77 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, sex characteristics, gender identity and expression, +level of experience, education, socio-economic status, nationality, personal +appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at [INSERT EMAIL ADDRESS]. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq + diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md new file mode 100644 index 0000000..f551fe6 --- /dev/null +++ b/.github/CONTRIBUTING.md @@ -0,0 +1,20 @@ +# Contributing + +TODO + +## Reporting an Issue + +TODO + +## Setup Development Environment + +TODO + +## Testing + +TODO + +## Opening a Pull Request + +TODO + diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000..0abd000 --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,27 @@ +(While submitting an issue briefly describe the problem you are facing or a new feature you would want to see added) + +## What happens? + +... + +## What were you expecting to happen? + +... + +## Steps to reproduce: + +* ... +* ... + +## Any errors, stacktrace, logs? + +... + +## Environment: + +* Runtime version(Java, Go, Python, etc): +* Desktop OS/version: + +## Additional comments: + +... diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..116890b --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,20 @@ +# Pull Request Template + +## Description + +Please provide a summary of the change and which issue it fixes. + +Fixes #(issue) + +## Type of change + +Please delete options that are not relevant. + +* Bug fix (non-breaking change which fixes an issue) +* New feature (non-breaking change which adds functionality) +* Breaking change (fix or feature that would cause existing functionality to not work as expected) +* This change requires a documentation update + +## Environment + +* Runtime version(Java, Go, Python, etc): diff --git a/.gitignore b/.gitignore index 258f7b1..6f82285 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ /vendor /build/bin +.idea +*.swp +.vscode diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..ec4f36c --- /dev/null +++ b/.travis.yml @@ -0,0 +1,20 @@ +--- +language: go +matrix: + include: + - go: 1.10.x + - go: 1.11.x + +before_install: + - go get github.com/golang/dep/cmd/dep + - go get golang.org/x/lint/golint + +install: + - # skip + +script: + - gofmt -d . + - golint $(go list ./... | grep -v /vendor/) + - go vet $(go list ./... | grep -v /vendor/) + - dep ensure + - go build diff --git a/ACKNOWLEDGEMENT.md b/ACKNOWLEDGEMENT.md new file mode 100644 index 0000000..69cf0ad --- /dev/null +++ b/ACKNOWLEDGEMENT.md @@ -0,0 +1,5 @@ +## Acknolwedgement + +Code base of git2consul.go tool is based on the excelent work done on [go-git2consul](https://github.com/Cimpress-MCP/go-git2consul) by [Calvin Leung Huang](https://github.com/calvn) under [Cimpress-MCP](https://github.com/Cimpress-MCP). + +git2consul.go was forked from the upstream from the following commit: [af9a2c80e699411f7559d7f6c3d29680c4ebd7b2](https://github.com/Cimpress-MCP/go-git2consul/commit/af9a2c80e699411f7559d7f6c3d29680c4ebd7b2) diff --git a/CHANGELOG.md b/CHANGELOG.md index 86d0bfa..e093c44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,17 @@ -## 0.1.0 (UNRELEASED) +## v0.1.0 +# +#### Summary +Release v0.1.0 covers the basic functionality with added features enhancing path management of the keys in the Consul KV Store and authentication for private repositories (f.e. GitHub Enterprise). +#### Features -* Initial version of go-git2consul that contains basic functionality -* Handle repository files to KV, no support for file extensions yet -* Handle tracking branches -* Interval and webhook polling -* Handle CRUD operations, and perform updates on deltas +* Added atomicity to the Consul transactions +* Added mountpoint option which sets the prefix for added keys +* Added skip_branch option which skipps the branch name for the added keys +* Expand YAML file content to k:v +* Added source_root option which allows to point the root of the repository from which we want to process the data +* Added authentication (basic and ssh) + +#### Bug fixes + +* Fix for transactions - limited to 64 elements chunks +* Fixed reference "not found" issue on branch pull diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..8e244cb --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,22 @@ +# Contributing to git2consul + +git2consul project is licensed under [Apache 2.0 license](LICENSE) and is open for contributions mainly via GitHub pull requests. + +## How to Contribute + +You can contribute to the project by submitting Pull Requests (PRs), submitting Issues that you found which will help us improve the git2consul, or by suggesting new features. + +When submitting a Pull Request, we expect that it will pass following requirements: + +- Your code must be written in an idiomatic Go. +- Formatted in accordance with the [gofmt](https://golang.org/cmd/gofmt). +- [go lint](https://github.com/golang/lint) shouldn't produce any warnings, same as [go vet](https://golang.org/cmd/vet) +- If you are submitting PR with a new feature, code should be covered with the suite of unit tests that test new functionality. Same rule applies for PRs that are bug fixes. + +### Commit message format + +Every commit message should contain information about package under which the changes were applied, short summary of the implemented changes and GitHub issue it relates to (if applicable): + +``` +: [Fixes #] +``` \ No newline at end of file diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json deleted file mode 100644 index 778e0d3..0000000 --- a/Godeps/Godeps.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "ImportPath": "github.com/Cimpress-MCP/go-git2consul", - "GoVersion": "go1.6", - "GodepVersion": "v62", - "Deps": [ - { - "ImportPath": "github.com/Sirupsen/logrus", - "Comment": "v0.9.0-17-ga26f435", - "Rev": "a26f43589d737684363ff856c5a0f9f24b946510" - }, - { - "ImportPath": "github.com/hashicorp/consul/api", - "Comment": "v0.6.4-241-g7637116", - "Rev": "763711686ae954b603d58a0980180ab5a910655d" - }, - { - "ImportPath": "github.com/hashicorp/go-cleanhttp", - "Rev": "ad28ea4487f05916463e2423a55166280e8254b5" - }, - { - "ImportPath": "github.com/hashicorp/serf/coordinate", - "Comment": "v0.7.0-58-gdefb069", - "Rev": "defb069b1bad9f7cdebc647810cb6ae398a1b617" - }, - { - "ImportPath": "golang.org/x/sys/unix", - "Rev": "f64b50fbea64174967a8882830d621a18ee1548e" - }, - { - "ImportPath": "gopkg.in/libgit2/git2go.v24", - "Rev": "8eaae73f85dd3df78df80d2dac066eb0866444ae" - } - ] -} diff --git a/Godeps/Readme b/Godeps/Readme deleted file mode 100644 index 4cdaa53..0000000 --- a/Godeps/Readme +++ /dev/null @@ -1,5 +0,0 @@ -This directory tree is generated automatically by godep. - -Please do not edit. - -See https://github.com/tools/godep for more information. diff --git a/Gopkg.lock b/Gopkg.lock new file mode 100644 index 0000000..14ec287 --- /dev/null +++ b/Gopkg.lock @@ -0,0 +1,274 @@ +# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + + +[[projects]] + name = "github.com/apex/log" + packages = [ + ".", + "handlers/discard", + "handlers/text" + ] + revision = "0296d6eb16bb28f8a0c55668affcf4876dc269be" + version = "v1.0.0" + +[[projects]] + name = "github.com/davecgh/go-spew" + packages = ["spew"] + revision = "346938d642f2ec3594ed81d874461961cd0faa76" + version = "v1.1.0" + +[[projects]] + name = "github.com/emirpasic/gods" + packages = [ + "containers", + "lists", + "lists/arraylist", + "trees", + "trees/binaryheap", + "utils" + ] + revision = "f6c17b524822278a87e3b3bd809fec33b51f5b46" + version = "v1.9.0" + +[[projects]] + branch = "master" + name = "github.com/gorilla/context" + packages = ["."] + revision = "08b5f424b9271eedf6f9f0ce86cb9396ed337a42" + +[[projects]] + name = "github.com/gorilla/mux" + packages = ["."] + revision = "e3702bed27f0d39777b0b37b664b6280e8ef8fbf" + version = "v1.6.2" + +[[projects]] + name = "github.com/hashicorp/consul" + packages = [ + "api", + "lib/freeport", + "testutil", + "testutil/retry" + ] + revision = "5174058f0d2bda63fa5198ab96c33d9a909c58ed" + version = "v1.1.0" + +[[projects]] + branch = "master" + name = "github.com/hashicorp/go-cleanhttp" + packages = ["."] + revision = "d5fe4b57a186c716b0e00b8c301cbd9b4182694d" + +[[projects]] + branch = "master" + name = "github.com/hashicorp/go-rootcerts" + packages = ["."] + revision = "6bb64b370b90e7ef1fa532be9e591a81c3493e00" + +[[projects]] + branch = "master" + name = "github.com/hashicorp/go-uuid" + packages = ["."] + revision = "27454136f0364f2d44b1276c552d69105cf8c498" + +[[projects]] + name = "github.com/hashicorp/serf" + packages = ["coordinate"] + revision = "d6574a5bb1226678d7010325fb6c985db20ee458" + version = "v0.8.1" + +[[projects]] + branch = "master" + name = "github.com/jbenet/go-context" + packages = ["io"] + revision = "d14ea06fba99483203c19d92cfcd13ebe73135f4" + +[[projects]] + name = "github.com/kevinburke/ssh_config" + packages = ["."] + revision = "9fc7bb800b555d63157c65a904c86a2cc7b4e795" + version = "0.4" + +[[projects]] + branch = "master" + name = "github.com/mitchellh/go-homedir" + packages = ["."] + revision = "b8bc1bf767474819792c23f32d8286a45736f1c6" + +[[projects]] + branch = "master" + name = "github.com/mitchellh/go-testing-interface" + packages = ["."] + revision = "a61a99592b77c9ba629d254a693acffaeb4b7e28" + +[[projects]] + name = "github.com/pelletier/go-buffruneio" + packages = ["."] + revision = "c37440a7cf42ac63b919c752ca73a85067e05992" + version = "v0.2.0" + +[[projects]] + branch = "master" + name = "github.com/pkg/errors" + packages = ["."] + revision = "e881fd58d78e04cf6d0de1217f8707c8cc2249bc" + +[[projects]] + name = "github.com/pmezard/go-difflib" + packages = ["difflib"] + revision = "792786c7400a136282c1664665ae0a8db921c6c2" + version = "v1.0.0" + +[[projects]] + name = "github.com/sergi/go-diff" + packages = ["diffmatchpatch"] + revision = "1744e2970ca51c86172c8190fadad617561ed6e7" + version = "v1.0.0" + +[[projects]] + name = "github.com/src-d/gcfg" + packages = [ + ".", + "scanner", + "token", + "types" + ] + revision = "f187355171c936ac84a82793659ebb4936bc1c23" + version = "v1.3.0" + +[[projects]] + name = "github.com/stretchr/testify" + packages = ["assert"] + revision = "12b6f73e6084dad08a7c6e575284b177ecafbc71" + version = "v1.2.1" + +[[projects]] + branch = "master" + name = "github.com/xanzy/ssh-agent" + packages = ["."] + revision = "ba9c9e33906f58169366275e3450db66139a31a9" + +[[projects]] + branch = "master" + name = "golang.org/x/crypto" + packages = [ + "cast5", + "curve25519", + "ed25519", + "ed25519/internal/edwards25519", + "internal/chacha20", + "openpgp", + "openpgp/armor", + "openpgp/elgamal", + "openpgp/errors", + "openpgp/packet", + "openpgp/s2k", + "poly1305", + "ssh", + "ssh/agent", + "ssh/knownhosts" + ] + revision = "80db560fac1fb3e6ac81dbc7f8ae4c061f5257bd" + +[[projects]] + branch = "master" + name = "golang.org/x/net" + packages = ["context"] + revision = "6078986fec03a1dcc236c34816c71b0e05018fda" + +[[projects]] + branch = "master" + name = "golang.org/x/sys" + packages = ["windows"] + revision = "bb729a57828d76e3050e664d86aa052741ab620f" + +[[projects]] + name = "golang.org/x/text" + packages = [ + "internal/gen", + "internal/triegen", + "internal/ucd", + "transform", + "unicode/cldr", + "unicode/norm" + ] + revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" + version = "v0.3.0" + +[[projects]] + name = "gopkg.in/src-d/go-billy.v4" + packages = [ + ".", + "helper/chroot", + "helper/polyfill", + "osfs", + "util" + ] + revision = "df053870ae7070b0350624ba5a22161ba3796cc0" + version = "v4.1.1" + +[[projects]] + name = "gopkg.in/src-d/go-git.v4" + packages = [ + ".", + "config", + "internal/revision", + "plumbing", + "plumbing/cache", + "plumbing/filemode", + "plumbing/format/config", + "plumbing/format/diff", + "plumbing/format/gitignore", + "plumbing/format/idxfile", + "plumbing/format/index", + "plumbing/format/objfile", + "plumbing/format/packfile", + "plumbing/format/pktline", + "plumbing/object", + "plumbing/protocol/packp", + "plumbing/protocol/packp/capability", + "plumbing/protocol/packp/sideband", + "plumbing/revlist", + "plumbing/storer", + "plumbing/transport", + "plumbing/transport/client", + "plumbing/transport/file", + "plumbing/transport/git", + "plumbing/transport/http", + "plumbing/transport/internal/common", + "plumbing/transport/server", + "plumbing/transport/ssh", + "storage", + "storage/filesystem", + "storage/filesystem/internal/dotgit", + "storage/memory", + "utils/binary", + "utils/diff", + "utils/ioutil", + "utils/merkletrie", + "utils/merkletrie/filesystem", + "utils/merkletrie/index", + "utils/merkletrie/internal/frame", + "utils/merkletrie/noder" + ] + revision = "57570e84f8c5739f0f4a59387493e590e709dde9" + version = "v4.4.0" + +[[projects]] + name = "gopkg.in/warnings.v0" + packages = ["."] + revision = "ec4a0fea49c7b46c2aeb0b51aac55779c607e52b" + version = "v0.1.2" + +[[projects]] + name = "gopkg.in/yaml.v2" + packages = ["."] + revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183" + version = "v2.2.1" + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + inputs-digest = "ae725478be1e950e93e820556b051a798057e1ec1d8839491ee84be615785604" + solver-name = "gps-cdcl" + solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml new file mode 100644 index 0000000..ecb4083 --- /dev/null +++ b/Gopkg.toml @@ -0,0 +1,27 @@ +[[constraint]] +name = "github.com/apex/log" +version = "~1.0.0" + +[[constraint]] +name = "github.com/gorilla/mux" +version = "~1.6.2" + +[[constraint]] +name = "github.com/hashicorp/consul" +version = "~1.1.0" + +[[constraint]] +name = "gopkg.in/src-d/go-git.v4" +version = "^4.4.0" + +[[constraint]] +name = "gopkg.in/yaml.v2" +version = "~2.2.1" + +[prune] +go-tests = true +unused-packages = true + +[[constraint]] + name = "github.com/stretchr/testify" + version = "1.2.1" diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/LICENSE.md b/LICENSE.md deleted file mode 100644 index c7ae93d..0000000 --- a/LICENSE.md +++ /dev/null @@ -1,7 +0,0 @@ -Copyright 2016 Cimpress - -Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/Makefile b/Makefile deleted file mode 100644 index 34a12a6..0000000 --- a/Makefile +++ /dev/null @@ -1,44 +0,0 @@ -TEST?=$(shell go list ./... | grep -v /vendor/) - -# Get git commit information -GIT_COMMIT=$(shell git rev-parse HEAD) -GIT_DIRTY=$(shell test -n "`git status --porcelain`" && echo "+CHANGES" || true) - -default: test - -test: generate - @echo " ==> Running tests..." - @go list $(TEST) \ - | grep -v "/vendor/" \ - | xargs -n1 go test -v -timeout=60s $(TESTARGS) -.PHONY: test - -generate: - @echo " ==> Generating..." - @find . -type f -name '.DS_Store' -delete - @go list ./... \ - | grep -v "/vendor/" \ - | xargs -n1 go generate $(PACKAGES) -.PHONY: generate - - -build: generate - @echo " ==> Building..." - @go build -ldflags "-X main.GitCommit=${GIT_COMMIT}${GIT_DIRTY}" . -.PHONY: build - -build-linux: create-build-image remove-dangling build-native -.PHONY: build-linux - -create-build-image: - @docker build -t cimpress/git2consul-builder $(CURDIR)/build/ -.PHONY: create-build-image - -remove-dangling: - @docker images --quiet --filter dangling=true | grep . | xargs docker rmi -.PHONY: remove-dangling - -run-build-image: - @echo " ===> Building..." - @docker run --rm --name git2consul-builder -v $(CURDIR):/app -v $(CURDIR)/build/bin:/build/bin --entrypoint /app/build/build.sh cimpress/git2consul-builder -.PHONY: run-build-image diff --git a/README.md b/README.md index 3ad7095..8357713 100644 --- a/README.md +++ b/README.md @@ -1,73 +1,228 @@ -# go-git2consul +# git2consul.go + +The git2consul.go tool is used to populate a [Consul](https://www.consul.io) key/value store from a git repo. + +The baseline source code was forked from [go-git2consul](https://github.com/Cimpress-MCP/go-git2consul) which was +inspired by the orginal [git2consul](https://github.com/breser/git2consul) tool. + +## Improvements Over NodeJS git2consul +* uses the official Consul Go Lang client library +* uses native Go Lang git implementation [go-git](https://github.com/src-d/go-git/) +* removal of nodejs and git runtime dependencies +* configuration is sourced locally instead of it being fetched from the Consul K/V +* transaction atomicity implies the set of keys is stored either entirely or not at all. Along with atomicity the number of the KV API calls is limited. However there is a pending [issue](https://github.com/hashicorp/consul/issues/2921) as Consul transaction endpoint can handle only 64 items in the payload. The transactions are executed in 64 elements chunks. + +## Installation + +git2consul.go comes in two variants: +* as a single binary file which after downloading can be placed in any working directory - either on the workstation (from which git2consul will be executed) or on the Consul node (depends whether access to the git repository is available from the Consul nodes or not) +* as a source code that can be build on the user workstation ([How to build from src?](#compiling-from-source)) + +## Documentation + +### Example +Simple example usage. + +``` +$ git2consul -config config.json -basic -user mygituser -password mygitpass -once +``` + +Simple example config file. +``` +{ + "repos": [ + { + "name": "example", + "url": "http://github.com/DummyOrg/ExampleRepo.git" + } + ] +} +``` + +### Command Line Options + +``` +$ git2consul -help +Usage of git2consul: + -basic + run with basic auth + -config string + path to config file + -debug + enable debugging mode + -key string + path to priv ssh key + -once + run git2consul once and exit + -password string + auth password + -ssh + run with ssh auth + -user string + auth user + -version + show version +``` + +### Configuration + +Configuration is provided with a JSON file and passed in via the `-config` flag. Repository +configuration will take care of cloning the repository into `local_store`, but it will not +be responsible for creating the actual `local_store` directory. Similarly, it is expected +that there is no collision of directory or file that contains the same name as the repository +name under `local_store`, or git2consul will exit with an error. If there is a git repository +under a specified repo name, and the origin URL is different from the one provided in the +configuration, it will be overwritten. + +#### Default configuration -[![Go Report Card](https://goreportcard.com/badge/github.com/Cimpress-MCP/go-git2consul)][goreport] +git2consul will attempt to use sane defaults for configuration. However, since git2consul needs to know which repository to pull from, minimal configuration is necessary. -[goreport]: https://goreportcard.com/report/github.com/Cimpress-MCP/go-git2consul -***NOTE: go-git2consul is experimental and still under development, and therefore should not be used in production!*** +| Configuration | Required | Default Value | Available Values | Description +|---------------------------|----------|----------------|--------------------------------------------| ----------- +| local_store | no | `os.TempDir()` | `string` | Local cache for git2consul to store its tracked repositories +| webhook:address | no | | `string` | Webhook listener address that git2consul will be using +| webhook:port | no | 9000 | `int` | Webhook listener port that git2consul will be using +| repos:name | yes | | `string` | Name of the repository. This will match the webhook path, if any are enabled +| repos:url | yes | | `string` | The URL of the repository +| repos:branches | no | master | `string` | Tracking branches of the repository +| repos:source_root | no | | `string` | Source root to apply on the repo. +| repos:expand_keys | no | | true, false | Enable/disable file content evaluation. +| repos:skip_branch_name | no | false | true, false | Enable/disable branch name pruning. +| repos:skip_repo_name | no | false | true, false | Enable/disable repository name pruning. +| repos:mount_point | no | | `string` | Sets the prefix which should be used for the path in the Consul KV Store +| repos:credentials:username | no | | `string` | Username for the Basic Auth +| repos:credentials:password | no | | `string` | Password/token for the Basic Auth +| repos:credentials:private_key:pk_key | no | | `string` | Path to the priv key used for the authentication +| repos:credentials:private_key:pk_username | no | git | `string` | Username used with the ssh authentication +| repos:credentials:private_key:pk_password | no | | `string` | Password used with the ssh authentication +| repos:hooks:type | no | polling | polling, github, stash, bitbucket, gitlab | Type of hook to use to fetch changes on the repository +| repos:hooks:interval | no | 60 | `int` | Interval, in seconds, to poll if polling is enabled +| repos:hooks:url | no | ?? | `string` | ??? +| consul:address | no | 127.0.0.1:8500 | `string` | Consul address to connect to. It can be either the IP or FQDN with port included +| consul:ssl | no | false | true, false | Whether to use HTTPS to communicate with Consul +| consul:ssl_verify | no | false | true, false | Whether to verify certificates when connecting via SSL +| consul:token | no | | `string` | Consul API Token + +### Webhooks -go-git2consul is a port of [git2consul](https://github.com/Cimpress-MCP/git2consul), which had great success and adoption. go-git2consul takes on the same basic principles as its predecessor, and attempts to improve upon some of its feature sets as well as add new ones. There are a few advantages to go-git2consul, including, but is not limited to, the use of the official Consul API and the removal of runtime dependencies such as node and git. +Webhooks will be served from a single port, and different repositories will be given different endpoints according to their name -Configuration on go-git2consul is sourced locally instead of it being fetched from the KV. This provides better isolation in cases where multiple instances of git2consul are running in order to provide high-availability, and addresses the issues mentioned in [Cimpress-MCP/git2consul#73](https://github.com/Cimpress-MCP/git2consul/issues/73). +Available endpoints: -## Configuration +* `:/{repository}/github` +* `:/{repository}/stash` +* `:/{repository}/bitbucket` +* `:/{repository}/gitlab` -Configuration is provided with a JSON file and passed in via the `-config` flag. Repository configuration will take care of cloning the repository into `local_store`, but it will not be responsible for creating the actual `local_store` directory. Similarly, it is expected that there is no collision of directory or file that contains the same name as the repository name under `local_store`, or git2consul will exit with an error. If there is a git repository under a specified repo name, and the origin URL is different from the one provided in the configuration, it will be overwritten. -### Default configuration +### Options -git2consul will attempt to use sane defaults for configuration. However, since git2consul needs to know which repository to pull from, minimal configuration is necessary. +#### source_root (default: undefined) -| Configuration | Required | Default Value | Available Values | Description -|----------------------|----------|----------------|--------------------------------------------| ----------- -| local_store | no | `os.TempDir()` | `string` | Local cache for git2consul to store its tracked repositories -| webhook:address | no | | `string` | Webhook listener address that git2consul will be using -| webhook:port | no | 9000 | `int` | Webhook listener port that git2consul will be using -| repos:name | yes | | `string` | Name of the repository. This will match the webhook path, if any are enabled -| repos:url | yes | | `string` | The URL of the repository -| repos:branches | no | master | `string` | Tracking branches of the repository -| repos:hooks:type | no | polling | polling, github, stash, bitbucket, gitlab | Type of hook to use to fetch changes on the repository -| repos:hooks:interval | no | 60 | `int` | Interval, in seconds, to poll if polling is enabled -| consul:address | no | 127.0.0.1:8500 | `string` | Consul address to connect to. It can be either the IP or FQDN with port included -| consul:ssl | no | false | true, false | Whether to use HTTPS to communicate with Consul -| consul:ssl_verify | no | false | true, false | Whether to verify certificates when connecting via SSL -| consul:token | no | | `string` | Consul API Token +The "source_root" instructs the app to navigate to the specified directory in the git repo making the value of source_root is trimed from the KV Store key. By default the entire repo is evaluated. -## Available command option flags +When you configure the source_root with `/top_level/lower_level` the file `/top_level/lower_level/foo/web.json` will be mapped to the KV store as `/foo/web.json` -### `-config` -The path to the configuration file. This flag is *required*. +#### mount_point (default: undefined) -### `-once` -Runs git2consul once and exits. This essentially ignores webhook polling. +The "mount_point" option sets the prefix for the path in the Consul KV Store under which the keys should be added. -### `-version` -Displays the version of git2consul and exits. All other commands are ignored. +#### expand_keys (default: undefined) -## Webhooks +The "expand_keys" instructs the app to evaluate known types of files. The content of the file is evaluated to key-value pair and pushed to the Consul KV store. -Webhooks will be served from a single port, and different repositories will be given different endpoints according to their name +##### Supported formats +* Text file - the file content is pushed to the KV store as it is. +* Yaml file - the file is evaluated into key-value pair. i.e `configuration.yml` +``` +--- +services: + apache: + port: 80 + ssh: + port: 22 +``` -Available endpoints: +will be evaluated to the following keys: +* `/configuration/services/apache/port` +* `/configuration/services/ssh/port` -* `:/{repository}/github` -* `:/{repository}/stash` -* `:/{repository}/bitbucket` -* `:/{repository}/gitlab` +#### skip_branch_name (default: false) + +The "skip_branch_name" instructs the app to prune the branch name. If set to true the branch name is pruned from the KV store key. + +#### skip_repo_name (default: false) + +The "skip_repo_name" instructs the app to prune the repository name. If set to true the repository name is pruned from the KV store key. + +#### credentials + +The "credentials" option provides the possibility to pass the credentials to authenticate to private git repositories. + +Sample config with basic auth (login:password/token) +``` +{ + "repos": [ + { + "name": "example", + "url": "http://github.com/DummyOrg/ExampleRepo.git", + "credentials: { + "username": "foo", + "password": "bar" + } + } + ] +} +``` +Sample config with ssh auth +``` +{ + "repos": [ + { + "name": "example", + "url": "http://github.com/DummyOrg/ExampleRepo.git", + "credentials: { + "private_key": { + "pk_key": "/path/to/priv_key", + "pk_username": "foo", + "pk_password": "bar" + } + } + } + ] +} +``` + +## Developing + +See [CONTRIBUTING.md](.github/CONTRIBUTING.md) for details. + +### Dependencies +* Go 1.10+ +* [dep](https://github.com/golang/dep) + +### Compiling From Source +``` +$ dep ensure +$ go build -o build/bin/git2consul +``` + +For Development/Debugging +``` +$ go build -gcflags='-N -l' -o build/bin/git2consul +``` + +## License + +See [LICENSE](LICENSE) for details. -## Future feature additions -* File format backend -* Support for source_root and mountpoint -* Support for tags as branches -* Support for Consul HTTP Basic Auth -* Logger support for other handlers other than text -* Auth support for webhooks banckends +## Acknowledgement +See [ACKNOWLEDGEMENT.md](ACKNOWLEDGEMENT.md) for details. -## Development dependencies -* Go 1.6 -* libgit2 v0.24.0 -* [glide](https://github.com/Masterminds/glide) +## Code of Conduct +See [CODE_OF_CONDUCT.md](.github/CODE_OF_CONDUCT.md) for details. -*Influenced by these awesome tools: git2consul, consul-replicate, fabio* diff --git a/TODO.md b/TODO.md deleted file mode 100644 index 9de954a..0000000 --- a/TODO.md +++ /dev/null @@ -1,71 +0,0 @@ -# TODO - -## Initial version requirements: -* [x] Better error handling of goroutines through errCh -* [x] Possible usage of a runner to abstract git operations from the consul package -* [x] Update on KV should be for modified and deleted files only -* [x] Switch from godep to glide -* [x] Switch to apex/log -* [x] Webhook polling - * [x] GitHub - * [x] Stash - * [x] Bitbucket - * [x] Gitlab -* [x] Accept consul configuration for the client -* [x] Add -once flag to run git2consul once -* [ ] Better CD/CI pipeline - * [ ] Cross-platform builds - * [ ] Travis/appveyor - -## Bugs/Issues: -* [x] Need to update diffs on the KV side - * [x] This includes only updating changed files - * [x] Delete untracked files -* [x] If repositories slice is empty, stop the program -* [x] Directory check has to check if it's a repository first -* [x] Runner, and watchers need a Stop() to handle cleanup better -* [x] Handle DoneCh better on both the watcher and runner -* [x] Handle initial load state better - * [x] Watcher should handle initial changes from load state - -## Error handling: -* [x] Better error handling on LoadRepos() - * [x] Bad configuration should be ignored -* [ ] Handle repository error with git reset or re-clone - -## Repo delta KV handling: -* [x] On added, modified: PUT KV -* [x] On delete: DEL KV -* [x] On rename: DEL old KV followed by PUT new KV - -## Test coverage -* [x] repository - * [x] New - * [x] Clone - * [x] Load - * [x] Pull - * [x] Diff - * [x] Ref - * [x] Checkout -* [x] config - * [x] Load -* [ ] runner - * [ ] New -* [ ] watcher - * [ ] Watcher - * [x] Interval - * [ ] Webhook -* [ ] kv - * [ ] Handler - * [ ] Branch - * [ ] KV - * [ ] InitHandler - * [ ] UpdateHandler - -Test suite enhancement: -* [ ] git-init on repo should be done on init() -* [ ] Setup and teardown for each test during - * [ ] Setup resets "remote" repo to initial commit - * [ ] Teardown cleans local store - -* Instead of testutil, we can use mocks to set up a mock repository.Repository object diff --git a/build/Dockerfile b/build/Dockerfile deleted file mode 100644 index 14d7a35..0000000 --- a/build/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -FROM alpine:edge -MAINTAINER Calvin Leung Huang - -RUN echo "@testing http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories - -RUN apk update -RUN apk --no-cache --no-progress add ca-certificates git go gcc musl-dev make cmake http-parser@testing perl \ - && rm -rf /var/cache/apk/* - -COPY . /build -WORKDIR /app - -RUN /build/configure.sh - -ENTRYPOINT ["/app/build/build.sh"] diff --git a/build/build.sh b/build/build.sh deleted file mode 100755 index 2893f86..0000000 --- a/build/build.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/sh -set -x -set -e - -# Set temp environment vars -export GOPATH=/tmp/go -export PATH=${PATH}:${GOPATH}/bin -export BUILDPATH=${GOPATH}/src/github.com/Cimpress-MCP/go-git2consul -export PKG_CONFIG_PATH="/usr/lib/pkgconfig:/usr/local/lib/pkgconfig" -export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:/tmp/libgit2/install/lib/pkgconfig:/tmp/openssl/install/lib/pkgconfig:/tmp/libssh2/build/src" - -FLAGS=$(pkg-config --static --libs --cflags libssh2 libgit2) || exit 1 -export CGO_LDFLAGS="/tmp/libgit2/build/libgit2.a /tmp/openssl/libcrypto.a /tmp/openssl/libssl.a /tmp/libssh2/build/src/libssh2.a -L/tmp/libgit2/build ${FLAGS}" -export CGO_CFLAGS="-I/tmp/libgit2/include" - -# Get git commit information -GIT_COMMIT=$(git rev-parse HEAD) -GIT_DIRTY=$(test -n "`git status --porcelain`" && echo "+CHANGES" || true) - -# Build git2consul -cd ${BUILDPATH} -go get -v -d -GOOS=linux GOARCH=amd64 go build -ldflags "-X main.GitCommit=${GIT_COMMIT}${GIT_DIRTY} -v -linkmode=external -extldflags '-static'" -o /build/bin/git2consul.linux.amd64 . -# GOOS=darwin GOARCH=amd64 CGO_ENABLED=1 go build -o /build/bin/git2consul.darwin.amd64 . diff --git a/build/configure.sh b/build/configure.sh deleted file mode 100755 index ae6f622..0000000 --- a/build/configure.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/sh -set -x -set -e - -# Set temp environment vars -export GOPATH=/tmp/go -export PATH=${PATH}:${GOPATH}/bin -export BUILDPATH=${GOPATH}/src/github.com/Cimpress-MCP/go-git2consul -export PKG_CONFIG_PATH="/usr/lib/pkgconfig/:/usr/local/lib/pkgconfig/" -export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:/tmp/libgit2/install/lib/pkgconfig:/tmp/openssl/install/lib/pkgconfig:/tmp/libssh2/build/src" - -# Install libraries -/build/install-openssl.sh -/build/install-libssh2.sh -/build/install-libgit2.sh - -# Set up go environment -mkdir -p $(dirname ${BUILDPATH}) -ln -s /app ${BUILDPATH} diff --git a/build/install-libgit2.sh b/build/install-libgit2.sh deleted file mode 100755 index d8f84a4..0000000 --- a/build/install-libgit2.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh -set -x - -# Set temp environment vars -export LIBGIT2REPO=https://github.com/libgit2/libgit2.git -export LIBGIT2BRANCH=v0.24.0 -export LIBGIT2PATH=/tmp/libgit2 -export PKG_CONFIG_PATH="/usr/lib/pkgconfig/:/usr/local/lib/pkgconfig/" -export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:/tmp/libgit2/install/lib/pkgconfig:/tmp/openssl/install/lib/pkgconfig:/tmp/libssh2/build/src" - -# Compile & Install libgit2 (v0.23) -git clone -b ${LIBGIT2BRANCH} --depth 1 -- ${LIBGIT2REPO} ${LIBGIT2PATH} - -mkdir -p ${LIBGIT2PATH}/build -cd ${LIBGIT2PATH}/build -cmake -DTHREADSAFE=ON \ - -DBUILD_CLAR=OFF \ - -DBUILD_SHARED_LIBS=OFF \ - -DCMAKE_C_FLAGS=-fPIC \ - -DCMAKE_BUILD_TYPE="RelWithDebInfo" \ - -DCMAKE_INSTALL_PREFIX=../install \ - .. -cmake --build . --target install - -# Cleanup -# rm -r ${LIBGIT2PATH} diff --git a/build/install-libssh2.sh b/build/install-libssh2.sh deleted file mode 100755 index e29139a..0000000 --- a/build/install-libssh2.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh -set -x - -# Set temp environment vars -export REPO=https://github.com/libssh2/libssh2 -export BRANCH=libssh2-1.7.0 -export REPO_PATH=/tmp/libssh2 -export PKG_CONFIG_PATH="/usr/lib/pkgconfig/:/usr/local/lib/pkgconfig/" -export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:/tmp/libgit2/install/lib/pkgconfig:/tmp/openssl/install/lib/pkgconfig:/tmp/libssh2/build/src" - -# Compile & Install libgit2 (v0.23) -git clone -b ${BRANCH} --depth 1 -- ${REPO} ${REPO_PATH} - -mkdir -p ${REPO_PATH}/build -cd ${REPO_PATH}/build -cmake -DTHREADSAFE=ON \ - -DBUILD_CLAR=OFF \ - -DBUILD_SHARED_LIBS=OFF \ - -DCMAKE_C_FLAGS=-fPIC \ - -DCMAKE_BUILD_TYPE="RelWithDebInfo" \ - -DCMAKE_INSTALL_PREFIX=../install \ - .. -cmake --build . --target install - -# Cleanup -# rm -r ${LIBGIT2PATH} diff --git a/build/install-openssl.sh b/build/install-openssl.sh deleted file mode 100755 index f9989f7..0000000 --- a/build/install-openssl.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/sh -set -x -set -e - -# Set temp environment vars -export REPO=https://github.com/openssl/openssl.git -export BRANCH=OpenSSL_1_0_2h -export BUILD_PATH=/tmp/openssl - -# Compile & Install libgit2 (v0.23) -git clone -b ${BRANCH} --depth 1 -- ${REPO} ${BUILD_PATH} - -mkdir -p ${BUILD_PATH}/install/lib -cd ${BUILD_PATH} -./config threads no-shared --prefix=${BUILD_PATH}/install -fPIC -DOPENSSL_PIC && -make depend && -make && -make install - -# Cleanup -# rm -r ${LIBGIT2PATH} diff --git a/config/config.go b/config/config.go index c95bb23..a395710 100644 --- a/config/config.go +++ b/config/config.go @@ -1,7 +1,37 @@ +/* +Copyright 2019 Kohl's Department Stores, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package config import "time" +// Credentials is the representation of git authentication +type Credentials struct { + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` + PrivateKey PrivateKey `json:"private_key,omitempty"` +} + +// PrivateKey is the representation of private key used for the authentication +type PrivateKey struct { + Key string `json:"pk_key"` + Username string `json:"pk_username,omitempty"` + Password string `json:"pk_password,omitempty"` +} + // Hook is the configuration for hooks type Hook struct { Type string `json:"type"` @@ -10,15 +40,21 @@ type Hook struct { Interval time.Duration `json:"interval"` // Specific to webhooks - Url string `json:"url,omitempty"` + URL string `json:"url,omitempty"` } // Repo is the configuration for the repository type Repo struct { - Name string `json:"name"` - Url string `json:"url"` - Branches []string `json:"branches"` - Hooks []*Hook `json:"hooks"` + Name string `json:"name"` + URL string `json:"url"` + Branches []string `json:"branches"` + Hooks []*Hook `json:"hooks"` + SourceRoot string `json:"source_root"` + MountPoint string `json:"mount_point"` + ExpandKeys bool `json:"expand_keys,omitempty"` + SkipBranchName bool `json:"skip_branch_name,omitempty"` + SkipRepoName bool `json:"skip_repo_name,omitempty"` + Credentials Credentials `json:"credentials,omitempty"` } // Config is used to represent the passed in configuration diff --git a/config/load.go b/config/load.go index 0a74458..cbd65d3 100644 --- a/config/load.go +++ b/config/load.go @@ -1,3 +1,19 @@ +/* +Copyright 2019 Kohl's Department Stores, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package config import ( @@ -5,6 +21,7 @@ import ( "fmt" "io/ioutil" "os" + "strings" "time" "github.com/apex/log" @@ -60,7 +77,7 @@ func (c *Config) checkConfig() error { } // Check on Url - if len(repo.Url) == 0 { + if len(repo.URL) == 0 { return fmt.Errorf("%s does no have a repository URL", repo.Name) } @@ -74,6 +91,26 @@ func (c *Config) checkConfig() error { return fmt.Errorf("Invalid interval: %s. Hook interval must be greater than zero", hook.Interval) } } + + // Check on mount_point + if len(repo.MountPoint) != 0 { + if strings.HasPrefix(repo.MountPoint, "/") { + return fmt.Errorf("Invalid mount point format for the %s repository - found \"/\" in the beginning of the path", repo.Name) + } + if !strings.HasSuffix(repo.MountPoint, "/") { + return fmt.Errorf("Invalid mount point format for the %s repository - missing trailing \"/\"", repo.Name) + } + } + + // Check on source_root + if len(repo.SourceRoot) != 0 { + if !strings.HasPrefix(repo.SourceRoot, "/") { + return fmt.Errorf("Invalid source_root format for the %s repository - missing \"/\" in the beginning of the path", repo.Name) + } + if !strings.HasSuffix(repo.SourceRoot, "/") { + return fmt.Errorf("Invalid source_root format for the %s repository - missing trailing \"/\"", repo.Name) + } + } } return nil diff --git a/config/load_test.go b/config/load_test.go index d8d11e2..33062b3 100644 --- a/config/load_test.go +++ b/config/load_test.go @@ -1,9 +1,27 @@ +/* +Copyright 2019 Kohl's Department Stores, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package config import ( "path/filepath" "testing" + "github.com/stretchr/testify/assert" + "github.com/apex/log" "github.com/apex/log/handlers/discard" ) @@ -16,16 +34,12 @@ func TestLoad(t *testing.T) { file := filepath.Join("test-fixtures", "local.json") _, err := Load(file) - if err != nil { - t.Fatal(err) - } + assert.NoError(t, err) } -func TestLoad_invalidConfig(t *testing.T) { +func TestLoadInvalidConfig(t *testing.T) { file := filepath.Join("test-fixtures", "invalid_config.json") _, err := Load(file) - if err == nil { - t.Fatal("Expected failure") - } + assert.Error(t, err) } diff --git a/config/mock/mock.go b/config/mock/mock.go index 734e182..6463c97 100644 --- a/config/mock/mock.go +++ b/config/mock/mock.go @@ -1,17 +1,34 @@ +/* +Copyright 2019 Kohl's Department Stores, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package mock import ( + "io/ioutil" "os" "time" - "github.com/Cimpress-MCP/go-git2consul/config" + "github.com/KohlsTechnology/git2consul-go/config" ) // RepoConfig returns a mock Repo config object -func RepoConfig(repoUrl string) *config.Repo { +func RepoConfig(repoURL string) *config.Repo { return &config.Repo{ Name: "git2consul-test-local", - Url: repoUrl, + URL: repoURL, Branches: []string{"master"}, Hooks: []*config.Hook{ { @@ -23,16 +40,20 @@ func RepoConfig(repoUrl string) *config.Repo { } // Config returns a mock Config object with one repository configuration -func Config(repoUrl string) *config.Config { +func Config(repoURL string) *config.Config { + localStore, err := ioutil.TempDir("", "git2consul-test-local") + if err != nil { + localStore = os.TempDir() + } return &config.Config{ - LocalStore: os.TempDir(), + LocalStore: localStore, HookSvr: &config.HookSvrConfig{ Port: 9000, }, Repos: []*config.Repo{ { Name: "git2consul-test-local", - Url: repoUrl, + URL: repoURL, Branches: []string{"master"}, Hooks: []*config.Hook{ { diff --git a/config/test-fixtures/test_ssh_key b/config/test-fixtures/test_ssh_key new file mode 100644 index 0000000..ada7da9 --- /dev/null +++ b/config/test-fixtures/test_ssh_key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA7nbOgvkvkYL7n4tdKxC4gLBaLEuOj9pqivX5N1jYgMkznvVy +9n4+vySMb5r/n4qTCDWasNjb/x5pXTwU3Bn43AwfikMyjseZkK7XLZRbXbo7mmVH +RS2RiOrGmkBnf0OMq+mZt1bRoyn2r1FdE89craHLDWjeNa/wz1si8GNZi0phxrLS +2tNlPEuXJ+c9WcQqNNiHr2anUWkNn0jvXxMZR3kGujbQA5JWjnP2blA9X41WRJTb +TcdnalFCiOpB4h342ehhZvDV4tBM3A/BMia00RxuLSgdaA6mK0RIrY0u9BFT4DtX +iI51wPdmbMn9mpLozTJ4oaEenYTY34FJQrY7oQIDAQABAoIBAHoILpJz9tXeU+pH +pXweaGzi0qKMX/5Z5eALFGgwfUsaq+IS9awS2wVyLLQIe/wnLCstU2Zg8+RQqdbe +okYovGPPiGx3Cu5qUrnp35Ahtcf3P/05Cwnp0ephSxUxFEXaSajIOLPMKCf5l5YM +azdyDJmGjKTmH6EV0oQqvWyLSw2uHAyiOMkz73F0ucCgjUIgVZOttAX7YXLe6Ab6 +3xHkmlzzlyaLxJJCNoQV9P1aFCWFiqz/7VLXN121NBikqP5tNQwqEiDpmPw5Uvff +NhHlI1rK0VLShJpSDAab3ojTHMJcREst1TIBaxDIFuwYcaaMTzos9nuD75Aw1L2C +DbG5PQkCgYEA+EzDT+BDTzXZzuYk5TZZY/a7gf/ePI5NQkf7vXu9qtHudaWXjF2X +CVtVQA9vf+mmG0KuKyq7Knhm4HcTWTO17nKf/Tm173ghagawHmodqixecXhisolC +YEZJS+RnkSkkkhSmbrFh3nQh9/GSb07d3bcapB7c2rAckJ9KdTfVencCgYEA9dv1 +TU6VNefhTvDYezMk+rKKFjDfi8QRUJX6XBfpyX7LAT2itLXKoHWpeXqQD8tpdyrq +vxOj2IQ1NF0YrcKtsHmEdjqqno9N93rIPaJJsy8B4/4oRVrNBLe/TYn6Uw2VtCqs +nXzLD9s2nm5l78s2GySIlOxUZrP1F1XLa8KlaKcCgYEAqAdajO1Y33uzv93izfJv +n8RSs/CxNg2enuITq/DXM4gJdTfwTJ7xHPXfxRAtBkTZkc0YDEJIkr8T74blYxIp +ZgnP1w983WdQRW3tNUfed1C7QNKVB/j3ICuwYllY9NUA2JJ85p/HeUDh0+Z4kDaw +0d+deb8g+iT/z6bcPmLgZpUCgYA8R0WJQq8KnGN9O0eYTR3P6V45upnUZqnoHB1Z +3vMO1+tlznrJ25hmZvK6OfaAKNsewIL1fhc5ypBQ2lJmp8h18BUt94xFe9UdzBi0 +I8n2CJxqDbJJ7s09Tt+0XxPksPv3RE81/Za7uH9XsLGFbUlCtl5WROsckxqQMhTB +wuNTGQKBgQDXccLNrawjHm/dCydSFADqcxJ1twQddKTGT2JVIIjanUeLdu1KvlAG +KO7SMbXHc85aaW3XCXnr9V1qhtLRrMiH3P/A5Li3g/KeeONln2/ruwQ/NJrFTtbM +QlkinJd7H+xtn9x+JBvJXyqiXqRB1J6Vf7/xj+8MaCDe6iauYeNDWg== +-----END RSA PRIVATE KEY----- diff --git a/glide.lock b/glide.lock deleted file mode 100644 index d72d5de..0000000 --- a/glide.lock +++ /dev/null @@ -1,28 +0,0 @@ -hash: fe700a632becb3b3104deb4c8cdb12c30fed7554bf7d2efc43eb240d93be43a2 -updated: 2016-05-24T17:32:28.314043612-04:00 -imports: -- name: github.com/apex/log - version: a999c1b29c986b1972ec53b15092b63a8051ec83 - subpackages: - - handlers/text -- name: github.com/gorilla/context - version: a8d44e7d8e4d532b6a27a02dd82abb31cc1b01bd -- name: github.com/gorilla/mux - version: 9c19ed558d5df4da88e2ade9c8940d742aef0e7e -- name: github.com/hashicorp/consul - version: 14c24154e8db989a8d17fd5e15f1b6ce7885f29e - subpackages: - - api -- name: github.com/hashicorp/go-cleanhttp - version: 875fb671b3ddc66f8e2f0acc33829c8cb989a38d -- name: github.com/hashicorp/serf - version: e4ec8cc423bbe20d26584b96efbeb9102e16d05f - subpackages: - - coordinate -- name: golang.org/x/sys - version: d4feaf1a7e61e1d9e79e6c4e76c6349e9cab0a03 - subpackages: - - unix -- name: gopkg.in/libgit2/git2go.v24 - version: 8eaae73f85dd3df78df80d2dac066eb0866444ae -devImports: [] diff --git a/glide.yaml b/glide.yaml deleted file mode 100644 index 39c4ea8..0000000 --- a/glide.yaml +++ /dev/null @@ -1,16 +0,0 @@ -package: github.com/Cimpress-MCP/go-git2consul -import: -- package: github.com/hashicorp/consul - subpackages: - - api -- package: github.com/hashicorp/go-cleanhttp -- package: github.com/hashicorp/serf - subpackages: - - coordinate -- package: golang.org/x/sys - subpackages: - - unix -- package: gopkg.in/libgit2/git2go.v24 -- package: github.com/apex/log - subpackages: - - handlers/text diff --git a/kv/api.go b/kv/api.go new file mode 100644 index 0000000..de1b97f --- /dev/null +++ b/kv/api.go @@ -0,0 +1,37 @@ +/* +Copyright 2019 Kohl's Department Stores, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kv + +import ( + "github.com/hashicorp/consul/api" + "github.com/KohlsTechnology/git2consul-go/repository" +) + +//Handler interface for Key-Value store. +type Handler interface { + PutKV(repository.Repo, string, []byte) error + DeleteKV(repository.Repo, string) error + DeleteTreeKV(repository.Repo, string) error + HandleUpdate(repository.Repo) error +} + +//API minimal Consul KV api implementation +type API interface { + Get(string, *api.QueryOptions) (*api.KVPair, *api.QueryMeta, error) + Put(*api.KVPair, *api.WriteOptions) (*api.WriteMeta, error) + Txn(api.KVTxnOps, *api.QueryOptions) (bool, *api.KVTxnResponse, *api.QueryMeta, error) +} diff --git a/kv/branch.go b/kv/branch.go index 9e34056..23aef49 100644 --- a/kv/branch.go +++ b/kv/branch.go @@ -1,30 +1,41 @@ +/* +Copyright 2019 Kohl's Department Stores, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package kv import ( - "io/ioutil" "os" - "path" "path/filepath" - "strings" - "github.com/Cimpress-MCP/go-git2consul/repository" "github.com/apex/log" - "github.com/hashicorp/consul/api" - "gopkg.in/libgit2/git2go.v24" + "github.com/KohlsTechnology/git2consul-go/repository" + "gopkg.in/src-d/go-git.v4/plumbing" ) // Push a repository branch to the KV // TODO: Optimize for PUT only on changes instead of the entire repo -func (h *KVHandler) putBranch(repo *repository.Repository, branch *git.Branch) error { +func (h *KVHandler) putBranch(repo repository.Repo, branch plumbing.ReferenceName) error { // Checkout branch - repo.CheckoutBranch(branch, &git.CheckoutOpts{ - Strategy: git.CheckoutForce, - }) + repo.CheckoutBranch(branch) // h, _ := repo.Head() // bn, _ := h.Branch().Name() // log.Debugf("(consul) pushBranch(): Branch: %s Head: %s", bn, h.Target().String()) - + workdir := repository.WorkDir(repo) + sourceRoot := repo.GetConfig().SourceRoot var pushFile = func(fullpath string, info os.FileInfo, err error) error { // Walk error if err != nil { @@ -41,37 +52,18 @@ func (h *KVHandler) putBranch(repo *repository.Repository, branch *git.Branch) e return nil } - // KV path, is repo/branch/file - branchName, err := branch.Name() + file := Init(fullpath, repo) + err = file.Create(h, repo) if err != nil { - return err + h.logger.Errorf("%s", err) } - - key := strings.TrimPrefix(fullpath, repo.Workdir()) - kvPath := path.Join(repo.Name(), branchName, key) - h.logger.Debugf("KV PUT changes: %s/%s: %s", repo.Name(), branchName, kvPath) - - data, err := ioutil.ReadFile(fullpath) - if err != nil { - return err - } - - p := &api.KVPair{ - Key: kvPath, - Value: data, - } - - _, err = h.Put(p, nil) - if err != nil { - return err - } - return nil } - - err := filepath.Walk(repo.Workdir(), pushFile) + workdir = filepath.Join(workdir, sourceRoot) + err := filepath.Walk(workdir, pushFile) if err != nil { log.WithError(err).Debug("PUT branch error") + return err } return nil diff --git a/kv/branch_test.go b/kv/branch_test.go new file mode 100644 index 0000000..0f04c28 --- /dev/null +++ b/kv/branch_test.go @@ -0,0 +1,69 @@ +/* +Copyright 2019 Kohl's Department Stores, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kv + +import ( + "io/ioutil" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/apex/log" + "github.com/stretchr/testify/assert" + "github.com/KohlsTechnology/git2consul-go/config" + "github.com/KohlsTechnology/git2consul-go/kv/mocks" + "github.com/KohlsTechnology/git2consul-go/repository" +) + +//TestPutBranch verifies putBranch function. +func TestPutBranch(t *testing.T) { + var repo repository.Repo + _, path, _, _ := runtime.Caller(0) + repo = &mocks.Repo{Path: filepath.Dir(path), Config: &config.Repo{}} + handler := &KVHandler{ + API: &mocks.KV{T: t}, + logger: log.WithFields(log.Fields{ + "caller": "consul", + }), + } + + handler.putBranch(repo, repo.Branch()) + handler.Commit() + + err := filepath.Walk(repository.WorkDir(repo), func(path string, f os.FileInfo, err error) error { + // Skip the .git directory + if f.IsDir() && f.Name() == ".git" { + return filepath.SkipDir + } + + // Do not push directories + if f.IsDir() { + return nil + } + + key := strings.TrimPrefix(path, repository.WorkDir(repo)) + kvPath := filepath.Join(repo.Name(), repo.Branch().Short(), key) + kvContent, _, err := handler.Get(kvPath, nil) + fileContent, err := ioutil.ReadFile(path) + + assert.Equal(t, fileContent, kvContent.Value) + return nil + }) + assert.NoError(t, err) +} diff --git a/kv/filehandler.go b/kv/filehandler.go new file mode 100644 index 0000000..222e77d --- /dev/null +++ b/kv/filehandler.go @@ -0,0 +1,176 @@ +/* +Copyright 2019 Kohl's Department Stores, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kv + +import ( + "io/ioutil" + "path/filepath" + "strconv" + "strings" + + "github.com/KohlsTechnology/git2consul-go/repository" + "gopkg.in/yaml.v2" +) + +//File interface to manipulate data from various types +//of files in the KV store. +type File interface { + Update(kv Handler, repo repository.Repo) error + Create(kv Handler, repo repository.Repo) error + Delete(kv Handler, repo repository.Repo) error + GetPath() string +} + +//TextFile structure +type TextFile struct { + path string +} + +//YAMLFile structure +type YAMLFile struct { + path string +} + +//Init initializes new instance of File interface based on it's extension. +func Init(path string, repo repository.Repo) File { + config := repo.GetConfig() + expandKeys := config.ExpandKeys + var f File + ext := filepath.Ext(path) + if expandKeys { + if ext == ".yml" { + f = &YAMLFile{path: path} + } + } + if f == nil { + f = &TextFile{path: path} + } + return f +} + +func getContent(f File) ([]byte, error) { + content, err := ioutil.ReadFile(f.GetPath()) + if err != nil { + return nil, err + } + return content, nil +} + +//GetPath returns the path to the file. +func (f *TextFile) GetPath() string { + return f.path +} + +//Create function creates the KV store entries based on the file content. +func (f *TextFile) Create(kv Handler, repo repository.Repo) error { + content, err := getContent(f) + if err != nil { + return err + } + err = kv.PutKV(repo, f.path, content) + if err != nil { + return err + } + return nil +} + +//Update functions updates the KV store based on the file content. +func (f *TextFile) Update(kv Handler, repo repository.Repo) error { + return f.Create(kv, repo) +} + +//Delete removes the key-value pair from the KV store. +func (f *TextFile) Delete(kv Handler, repo repository.Repo) error { + err := kv.DeleteKV(repo, f.path) + if err != nil { + return err + } + return nil +} + +//Create function creates the KV store entries based on the file content. +func (f *YAMLFile) Create(kv Handler, repo repository.Repo) error { + content, err := getContent(f) + if err != nil { + return err + } + yamlTree := make(map[interface{}]interface{}) + err = yaml.Unmarshal(content, &yamlTree) + if err != nil { + return err + } + path := f.GetPath() + extension := filepath.Ext(path) + fileName := strings.TrimSuffix(path, extension) + for key, value := range entriesToKV(yamlTree) { + err = kv.PutKV(repo, filepath.Join(fileName, key), value) + if err != nil { + return err + } + } + return nil +} + +//Update functions updates the KV store based on the file content. +func (f *YAMLFile) Update(kv Handler, repo repository.Repo) error { + f.Delete(kv, repo) + return f.Create(kv, repo) +} + +//Delete removes the key-value pairs from the KV store under given prefix. +func (f *YAMLFile) Delete(kv Handler, repo repository.Repo) error { + path := f.GetPath() + extension := filepath.Ext(path) + fileName := strings.TrimSuffix(path, extension) + err := kv.DeleteTreeKV(repo, fileName) + if err != nil { + return err + } + return nil +} + +//GetPath returns the path to the file. +func (f *YAMLFile) GetPath() string { + return f.path +} + +func entriesToKV(node map[interface{}]interface{}) map[string][]byte { + keys := make(map[string][]byte) + for key, value := range node { + switch value.(type) { + case string: + keys[key.(string)] = []byte(value.(string)) + case int: + keys[key.(string)] = []byte(strconv.Itoa(value.(int))) + case bool: + keys[key.(string)] = []byte(strconv.FormatBool(value.(bool))) + case float64: + keys[key.(string)] = []byte(strconv.FormatFloat(value.(float64), 'f', 2, 64)) + case map[interface{}]interface{}: + for k, v := range entriesToKV(value.(map[interface{}]interface{})) { + keys[filepath.Join(key.(string), k)] = v + } + case []interface{}: + for index, item := range value.([]interface{}) { + for k, v := range entriesToKV(item.(map[interface{}]interface{})) { + keys[filepath.Join(key.(string), strconv.Itoa(index), k)] = v + } + } + } + } + return keys +} diff --git a/kv/filehandler_test.go b/kv/filehandler_test.go new file mode 100644 index 0000000..5cc3054 --- /dev/null +++ b/kv/filehandler_test.go @@ -0,0 +1,175 @@ +/* +Copyright 2019 Kohl's Department Stores, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kv + +import ( + "bytes" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/KohlsTechnology/git2consul-go/repository" + + yaml "gopkg.in/yaml.v2" +) + +type mockHandler struct { + t *testing.T + filePath string +} + +var ( + yamlFile File + textFile File + handler Handler + yamlTree map[interface{}]interface{} + keys map[string][]byte +) + +const ( + content = "---\nei_unix_cavisson::cavisson_collector_srv: 10.206.96.18\n" + + "ei_unix_cavisson::cavisson_port: 7891\n" + + "ei_unix_cavisson::cavisson_java_home: \"/etc/alternatives/jre_openjdk\"\n" + + "dict:\n" + + " key_1: value_1\n" + + " key_2:\n" + + " - first_elem:\n" + + " key_3: true\n" + + " key_4: 2.35\n" + + " - second_element: value_4\n" +) + +//TestFile performs tests on implemented file handlers. +// * yaml +// * text +func TestFileHandler(t *testing.T) { + var repo repository.Repo + yamlTree = make(map[interface{}]interface{}) + err := yaml.Unmarshal([]byte(content), &yamlTree) + if err != nil { + t.Fatal(err) + } + filePath := filepath.Join(os.TempDir(), "foo.yml") + defer os.Remove(filePath) + err = ioutil.WriteFile(filePath, []byte(content), 0700) + if err != nil { + t.Fatal(err) + } + yamlFile = &YAMLFile{filePath} + textFile = &TextFile{filePath} + handler = &mockHandler{ + t: t, + filePath: filePath, + } + t.Run("TestParseYAMLFile", testParseYamlEntries) + t.Run("TestCreateYAMLFile", func(t *testing.T) { testCreateYAMLFile(t, repo) }) + t.Run("TestDeleteYAMLFile", func(t *testing.T) { testDeleteYAMLFile(t, repo) }) + t.Run("TestCreateTextFile", func(t *testing.T) { testCreateTextFile(t, repo) }) + t.Run("TestDeleteTextFile", func(t *testing.T) { testDeleteTextFile(t, repo) }) +} + +//testParsNodes verfies yaml file evaluation function. +func testParseYamlEntries(t *testing.T) { + keys := entriesToKV(yamlTree) + if string(keys["ei_unix_cavisson::cavisson_collector_srv"]) != "10.206.96.18" { + t.Fatal("Missing key or invalid value") + } + if string(keys["ei_unix_cavisson::cavisson_port"]) != "7891" { + t.Fatal("Missing key or invalid value") + } + if string(keys["dict/key_1"]) != "value_1" { + t.Fatal("Missing key or invalid value") + } + if string(keys["dict/key_2/0/first_elem/key_3"]) != "true" { + t.Fatal("Missing key or invalid value") + } + if string(keys["dict/key_2/1/second_element"]) != "value_4" { + t.Fatal("Missing key or invalid value") + } + if string(keys["dict/key_2/0/first_elem/key_4"]) != "2.35" { + t.Fatal("Missing key or invalid value") + } +} + +func testCreateYAMLFile(t *testing.T, repo repository.Repo) { + keys = make(map[string][]byte) + ext := filepath.Ext(yamlFile.GetPath()) + yamlPath := strings.TrimRight(yamlFile.GetPath(), ext) + err := yamlFile.Create(handler, repo) + if err != nil { + t.Fatal(err) + } + if len(keys) == 0 { + t.Fatalf("Keys empty: %+v", keys) + } + for k, v := range entriesToKV(yamlTree) { + if bytes.Compare(keys[filepath.Join(yamlPath, k)], v) == 0 { + delete(keys, filepath.Join(yamlPath, k)) + } + } + if len(keys) != 0 { + t.Fatalf("Keys not empty: %+v", keys) + } +} + +func testDeleteYAMLFile(t *testing.T, repo repository.Repo) { + err := yamlFile.Delete(handler, repo) + assert.NoError(t, err) +} + +func testCreateTextFile(t *testing.T, repo repository.Repo) { + keys = make(map[string][]byte) + textPath := textFile.GetPath() + err := textFile.Create(handler, repo) + assert.NoError(t, err) + assert.Len(t, keys, 1) + assert.Equal(t, keys[textPath], []byte(content)) +} + +func testDeleteTextFile(t *testing.T, repo repository.Repo) { + err := textFile.Delete(handler, repo) + assert.NoError(t, err) +} + +func (a mockHandler) PutKV(repo repository.Repo, path string, content []byte) error { + keys[path] = content + return nil +} + +func (a mockHandler) DeleteKV(repo repository.Repo, path string) error { + if a.filePath != path { + return fmt.Errorf("%s differs from %s", a.filePath, path) + } + return nil +} + +func (a mockHandler) DeleteTreeKV(repo repository.Repo, path string) error { + filePath := strings.TrimSuffix(a.filePath, filepath.Ext(a.filePath)) + if filePath != path { + return fmt.Errorf("%s differs from %s", a.filePath, path) + } + return nil +} + +func (a mockHandler) HandleUpdate(repo repository.Repo) error { + return nil +} diff --git a/kv/handler.go b/kv/handler.go index d2b955a..572d4fb 100644 --- a/kv/handler.go +++ b/kv/handler.go @@ -1,20 +1,45 @@ +/* +Copyright 2019 Kohl's Department Stores, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package kv import ( - "crypto/tls" - "net/http" + "fmt" - "github.com/Cimpress-MCP/go-git2consul/config" "github.com/apex/log" "github.com/hashicorp/consul/api" + "github.com/KohlsTechnology/git2consul-go/config" ) +const consulTxnSize = 64 + // KVHandler is used to manipulate the KV type KVHandler struct { - *api.KV + API + api.KVTxnOps logger *log.Entry } +//TransactionIntegrityError implements error to handle any violation of transaction atomicity. +type TransactionIntegrityError struct { + msg string +} + +func (e *TransactionIntegrityError) Error() string { return e.msg } + // New creates new KV handler to manipulate the Consul VK func New(config *config.ConsulConfig) (*KVHandler, error) { client, err := newAPIClient(config) @@ -29,8 +54,9 @@ func New(config *config.ConsulConfig) (*KVHandler, error) { kv := client.KV() handler := &KVHandler{ - KV: kv, - logger: logger, + API: kv, + KVTxnOps: nil, + logger: logger, } return handler, nil @@ -52,11 +78,7 @@ func newAPIClient(config *config.ConsulConfig) (*api.Client, error) { } if !config.SSLVerify { - consulConfig.HttpClient.Transport = &http.Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - } + consulConfig.TLSConfig.InsecureSkipVerify = true } client, err := api.NewClient(consulConfig) @@ -66,3 +88,85 @@ func newAPIClient(config *config.ConsulConfig) (*api.Client, error) { return client, nil } + +//Put overrides Consul API Put function to add entry to KVTxnOps. +func (h *KVHandler) Put(kvPair *api.KVPair, wOptions *api.WriteOptions) (*api.WriteMeta, error) { + txnItem := &api.KVTxnOp{ + Verb: api.KVSet, + Key: kvPair.Key, + Value: kvPair.Value, + } + h.KVTxnOps = append(h.KVTxnOps, txnItem) + return nil, nil +} + +//Delete overrides Consul API Delete function to add entry to KVTxnOps. +func (h *KVHandler) Delete(key string, wOptions *api.WriteOptions) (*api.WriteMeta, error) { + txnItem := &api.KVTxnOp{ + Verb: api.KVDelete, + Key: key, + } + h.KVTxnOps = append(h.KVTxnOps, txnItem) + return nil, nil +} + +//DeleteTree overrides Consul API DeleteTree function to add entry to KVTxnOps. +func (h *KVHandler) DeleteTree(key string, wOptions *api.WriteOptions) (*api.WriteMeta, error) { + txnItem := &api.KVTxnOp{ + Verb: api.KVDeleteTree, + Key: key, + } + h.KVTxnOps = append(h.KVTxnOps, txnItem) + return nil, nil +} + +//Commit function executes set of operations from KVTxnOps as single transaction. +func (h *KVHandler) Commit() error { + defer func() { + h.KVTxnOps = nil + }() + var kvTxnOps = h.KVTxnOps + //move modify index check to the end + if h.KVTxnOps[0].Verb == api.KVCheckIndex { + length := len(h.KVTxnOps) + kvTxnOps = append(h.KVTxnOps[1:length-1], h.KVTxnOps[0], h.KVTxnOps[length-1]) + } + for _, slice := range h.splitIntoSlices(kvTxnOps, consulTxnSize) { + err := h.executeTransaction(slice) + if err != nil { + return err + } + } + return nil +} + +func (h *KVHandler) executeTransaction(KVTxnOps api.KVTxnOps) error { + status, response, _, err := h.Txn(KVTxnOps, nil) + if err != nil { + return err + } + h.logger.Debugf("Transaction with %d items was sent to the KV store", len(KVTxnOps)) + if !status { + errMsg := "" + for _, txError := range response.Errors { + errMsg += fmt.Sprintf("%s\n", txError.What) + } + return &TransactionIntegrityError{fmt.Sprintf("Transaction has been rolled back due to: %s", errMsg)} + } + return nil +} + +func (h *KVHandler) splitIntoSlices(kvTxnOps api.KVTxnOps, sliceLength int) []api.KVTxnOps { + var kvTxnSlices []api.KVTxnOps + for len(kvTxnOps) > 0 { + index := sliceLength + if index > len(kvTxnOps) { + index = len(kvTxnOps) + } + var slice api.KVTxnOps + slice = append(slice, kvTxnOps[:index]...) + kvTxnOps = kvTxnOps[index:] + kvTxnSlices = append(kvTxnSlices, slice) + } + return kvTxnSlices +} diff --git a/kv/init.go b/kv/init.go index 3748f83..4fc5eb6 100644 --- a/kv/init.go +++ b/kv/init.go @@ -1,12 +1,33 @@ +/* +Copyright 2019 Kohl's Department Stores, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package kv import ( - "github.com/Cimpress-MCP/go-git2consul/repository" - "gopkg.in/libgit2/git2go.v24" + "path/filepath" + "strings" + + "github.com/KohlsTechnology/git2consul-go/repository" + "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/object" + "gopkg.in/src-d/go-git.v4/utils/merkletrie" ) // HandleInit handles initial fetching of the KV on start -func (h *KVHandler) HandleInit(repos []*repository.Repository) error { +func (h *KVHandler) HandleInit(repos []repository.Repo) error { for _, repo := range repos { err := h.handleRepoInit(repo) if err != nil { @@ -19,47 +40,44 @@ func (h *KVHandler) HandleInit(repos []*repository.Repository) error { // Handles differences on all branches of a repository, comparing the ref // of the branch against the one in the KV -func (h *KVHandler) handleRepoInit(repo *repository.Repository) error { +func (h *KVHandler) handleRepoInit(repo repository.Repo) error { repo.Lock() defer repo.Unlock() - itr, err := repo.NewReferenceIterator() + storer := repo.GetStorer() + itr, err := storer.IterReferences() if err != nil { return err } - defer itr.Free() - // Handle all local refs for { ref, err := itr.Next() if err != nil { break } - - b, err := ref.Branch().Name() - if err != nil { - return err + if strings.Contains(ref.String(), "HEAD") { + continue } - // Get only local refs - if ref.IsRemote() == false { - h.logger.Infof("KV GET ref: %s/%s", repo.Name(), b) - kvRef, err := h.getKVRef(repo, b) + if ref.Name().IsRemote() == false { + h.logger.Infof("KV GET ref: %s/%s", repo.Name(), ref.Name()) + kvRef, err := h.getKVRef(repo, ref.Name().String()) + if err != nil { return err } - localRef := ref.Target().String() + localRef := ref.Hash().String() if len(kvRef) == 0 { // There is no ref in the KV, push the entire branch - h.logger.Infof("KV PUT changes: %s/%s", repo.Name(), b) - h.putBranch(repo, ref.Branch()) + h.logger.Infof("KV PUT changes: %s/%s", repo.Name(), ref.Name()) + h.putBranch(repo, plumbing.ReferenceName(ref.Name().Short())) - h.logger.Infof("KV PUT ref: %s/%s", repo.Name(), b) - h.putKVRef(repo, b) + h.logger.Infof("KV PUT ref: %s/%s", repo.Name(), ref.Name()) + h.putKVRef(repo, ref.Name().String()) } else if kvRef != localRef { - // Check if the ref belongs to that repo + //Check if the ref belongs to that repo err := repo.CheckRef(kvRef) if err != nil { return err @@ -72,46 +90,47 @@ func (h *KVHandler) handleRepoInit(repo *repository.Repository) error { } h.handleDeltas(repo, deltas) - err = h.putKVRef(repo, b) + err = h.putKVRef(repo, ref.Name().String()) if err != nil { return err } - h.logger.Debugf("KV PUT ref: %s/%s", repo.Name(), b) + h.logger.Debugf("KV PUT ref: %s/%s", repo.Name(), ref.Name()) } } } - return nil } // Helper function that handles deltas -func (h *KVHandler) handleDeltas(repo *repository.Repository, deltas []git.DiffDelta) error { - // Handle modified and deleted files - for _, d := range deltas { - switch d.Status { - case git.DeltaRenamed: - h.logger.Debugf("Detected renamed file: %s", d.NewFile.Path) - h.logger.Infof("KV DEL %s/%s/%s", repo.Name(), repo.Branch(), d.OldFile.Path) - err := h.deleteKV(repo, d.OldFile.Path) - if err != nil { - return err - } - h.logger.Infof("KV PUT %s/%s/%s", repo.Name(), repo.Branch(), d.NewFile.Path) - err = h.putKV(repo, d.NewFile.Path) +func (h *KVHandler) handleDeltas(repo repository.Repo, diff object.Changes) error { + for _, d := range diff { + action, err := d.Action() + if err != nil { + return err + } + workDir := repository.WorkDir(repo) + switch action { + case merkletrie.Insert: + filePath := filepath.Join(workDir, d.To.Name) + h.logger.Debugf("Detected added file: %s", filePath) + file := Init(filePath, repo) + err := file.Create(h, repo) if err != nil { return err } - case git.DeltaAdded, git.DeltaModified: - h.logger.Debugf("Detected added/modified file: %s", d.NewFile.Path) - h.logger.Infof("KV PUT %s/%s/%s", repo.Name(), repo.Branch(), d.NewFile.Path) - err := h.putKV(repo, d.NewFile.Path) + case merkletrie.Modify: + filePath := filepath.Join(workDir, d.To.Name) + h.logger.Debugf("Detected modified file: %s", filePath) + file := Init(filePath, repo) + err := file.Update(h, repo) if err != nil { return err } - case git.DeltaDeleted: - h.logger.Debugf("Detected deleted file: %s", d.OldFile.Path) - h.logger.Infof("KV DEL %s/%s/%s", repo.Name(), repo.Branch(), d.OldFile.Path) - err := h.deleteKV(repo, d.OldFile.Path) + case merkletrie.Delete: + filePath := filepath.Join(workDir, d.From.Name) + h.logger.Debugf("Detected deleted file: %s", filePath) + file := Init(filePath, repo) + err := file.Delete(h, repo) if err != nil { return err } diff --git a/kv/kv.go b/kv/kv.go index 0e07281..833e2f5 100644 --- a/kv/kv.go +++ b/kv/kv.go @@ -1,32 +1,48 @@ +/* +Copyright 2019 Kohl's Department Stores, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package kv import ( - "io/ioutil" - "path" - "path/filepath" - - "github.com/Cimpress-MCP/go-git2consul/repository" "github.com/hashicorp/consul/api" + "github.com/KohlsTechnology/git2consul-go/repository" ) -func (h *KVHandler) putKV(repo *repository.Repository, prefix string) error { +// PutKV triggers an KV api request to put data to the Consul. +func (h *KVHandler) PutKV(repo repository.Repo, prefix string, value []byte) error { head, err := repo.Head() if err != nil { return err } - branchName, err := head.Branch().Name() - if err != nil { - return err - } + branchName := head.Name().Short() - key := path.Join(repo.Name(), branchName, prefix) - filePath := filepath.Join(repo.Workdir(), prefix) - value, err := ioutil.ReadFile(filePath) + key, status, err := getItemKey(repo, prefix) if err != nil { - return err + if status == SourceRootNotInPrefix { + h.logger.Infof("%s Skipping!", err) + } + if status == PathFormatterError { + return err + } + return nil } + h.logger.Debugf("KV PUT: %s/%s: %s", repo.Name(), branchName, key) + p := &api.KVPair{ Key: key, Value: value, @@ -40,20 +56,43 @@ func (h *KVHandler) putKV(repo *repository.Repository, prefix string) error { return nil } -func (h *KVHandler) deleteKV(repo *repository.Repository, prefix string) error { - head, err := repo.Head() +//DeleteKV deletes provided item from the KV store. +func (h *KVHandler) DeleteKV(repo repository.Repo, prefix string) error { + key, status, err := getItemKey(repo, prefix) if err != nil { - return err + if status == SourceRootNotInPrefix { + h.logger.Infof("%s Skipping!", err) + } + if status == PathFormatterError { + return err + } + return nil } - branchName, err := head.Branch().Name() + h.logger.Infof("KV DEL %s/%s/%s", repo.Name(), repo.Branch(), key) + _, err = h.Delete(key, nil) if err != nil { return err } - key := path.Join(repo.Name(), branchName, prefix) + return nil +} - _, err = h.Delete(key, nil) +//DeleteTreeKV deletes recursively all the keys with given prefix. +func (h *KVHandler) DeleteTreeKV(repo repository.Repo, prefix string) error { + key, status, err := getItemKey(repo, prefix) + if err != nil { + if status == SourceRootNotInPrefix { + h.logger.Infof("%s Skipping!", err) + } + if status == PathFormatterError { + return err + } + return nil + } + + h.logger.Infof("KV DEL %s/%s/%s", repo.Name(), repo.Branch(), key) + _, err = h.DeleteTree(key, nil) if err != nil { return err } diff --git a/kv/kv_test.go b/kv/kv_test.go new file mode 100644 index 0000000..740a9c3 --- /dev/null +++ b/kv/kv_test.go @@ -0,0 +1,108 @@ +/* +Copyright 2019 Kohl's Department Stores, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kv + +import ( + "fmt" + "io/ioutil" + "os" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/apex/log" + "github.com/KohlsTechnology/git2consul-go/config" + "github.com/KohlsTechnology/git2consul-go/kv/mocks" + "github.com/KohlsTechnology/git2consul-go/repository" +) + +//TestKV runs a test against KVPUT and DeleteKV handler functions. +func TestKV(t *testing.T) { + handler := &KVHandler{ + API: &mocks.KV{T: t}, + logger: log.WithFields(log.Fields{ + "caller": "consul", + })} + repoPath, err := ioutil.TempDir("", "local-repo") + defer os.RemoveAll(repoPath) + assert.NoError(t, err) + repo := &mocks.Repo{Path: repoPath, Config: &config.Repo{}, T: t} + repoPath = repository.WorkDir(repo) + t.Run("testPutKV", func(t *testing.T) { testPutKV(t, repo, handler) }) + t.Run("testDeleteKV", func(t *testing.T) { testDeleteKV(t, repo, handler) }) +} + +//testPutKV verifies the data pushed by putKV function. +func testPutKV(t *testing.T, repo repository.Repo, handler *KVHandler) { + f, err := ioutil.TempFile(repository.WorkDir(repo), "example.txt") + f.Write([]byte("Example content")) + f.Close() + assert.NoError(t, err) + value, err := ioutil.ReadFile(f.Name()) + + assert.NoError(t, err) + prefix := strings.TrimPrefix(f.Name(), repository.WorkDir(repo)) + err = handler.PutKV(repo, prefix, value) + if err != nil { + t.Fatal(err) + } + err = handler.Commit() + assert.NoError(t, err) + + head, err := repo.Head() + assert.NoError(t, err) + + pair, _, err := handler.Get(fmt.Sprintf("%s/%s%s", repo.Name(), head.Name().Short(), prefix), nil) + assert.NoError(t, err) + + if assert.NotNil(t, pair) { + assert.Equal(t, value, pair.Value) + } +} + +//testDeleteKV ensures data has been deleted. +func testDeleteKV(t *testing.T, repo repository.Repo, handler *KVHandler) { + f, err := ioutil.TempFile(repository.WorkDir(repo), "example.txt") + f.Write([]byte("Example content to delete")) + f.Close() + assert.NoError(t, err) + value, err := ioutil.ReadFile(f.Name()) + + prefix := strings.TrimPrefix(f.Name(), repository.WorkDir(repo)) + err = handler.PutKV(repo, prefix, value) + assert.NoError(t, err) + + err = handler.Commit() + assert.NoError(t, err) + + head, err := repo.Head() + assert.NoError(t, err) + + pair, _, err := handler.Get(fmt.Sprintf("%s/%s%s", repo.Name(), head.Name().Short(), prefix), nil) + assert.NoError(t, err) + assert.NotNil(t, pair) + + handler.DeleteKV(repo, prefix) + err = handler.Commit() + assert.NoError(t, err) + + pair, _, err = handler.Get(fmt.Sprintf("%s/%s%s", repo.Name(), head.Name().Short(), prefix), nil) + assert.NoError(t, err) + + assert.Nil(t, pair) +} diff --git a/kv/mocks/kv.go b/kv/mocks/kv.go new file mode 100644 index 0000000..ae59e0f --- /dev/null +++ b/kv/mocks/kv.go @@ -0,0 +1,78 @@ +/* +Copyright 2019 Kohl's Department Stores, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mocks + +import ( + "math/rand" + "testing" + + "github.com/hashicorp/consul/api" +) + +type item struct { + value []byte + modifyindex uint64 +} + +type KV struct { + T *testing.T + items map[string]*item +} + +func (kv *KV) Get(key string, opts *api.QueryOptions) (*api.KVPair, *api.QueryMeta, error) { + kv.T.Logf("KV Get %s", key) + if val, ok := kv.items[key]; ok { + return &api.KVPair{Key: key, Value: val.value, ModifyIndex: val.modifyindex}, nil, nil + } + return nil, nil, nil +} + +func (kv *KV) Put(kvPair *api.KVPair, wOptions *api.WriteOptions) (*api.WriteMeta, error) { + if kv.items == nil { + kv.items = make(map[string]*item) + } + kv.T.Logf("KV Put %s", kvPair.Key) + kv.items[kvPair.Key] = &item{value: kvPair.Value, modifyindex: rand.Uint64()} + return nil, nil +} + +func (kv *KV) Delete(key string, wOptions *api.WriteOptions) (*api.WriteMeta, error) { + delete(kv.items, key) + return nil, nil +} + +func (kv *KV) Txn(txnops api.KVTxnOps, opts *api.QueryOptions) (bool, *api.KVTxnResponse, *api.QueryMeta, error) { + var checkIndexItem *api.KVTxnOp + if length := len(txnops); length > 1 && txnops[length-2].Verb == api.KVCheckIndex { + checkIndexItem = txnops[length-2] + } + for _, item := range txnops { + switch item.Verb { + case api.KVSet: + if checkIndexItem != nil && item.Key == checkIndexItem.Key { + kvPair, _, _ := kv.Get(item.Key, nil) + if kvPair.ModifyIndex != checkIndexItem.Index { + return false, &api.KVTxnResponse{}, nil, nil + } + } + kv.Put(&api.KVPair{Key: item.Key, Value: item.Value}, nil) + case api.KVDelete: + kv.Delete(item.Key, nil) + } + } + return true, nil, nil, nil +} diff --git a/kv/mocks/repository.go b/kv/mocks/repository.go new file mode 100644 index 0000000..dd1163f --- /dev/null +++ b/kv/mocks/repository.go @@ -0,0 +1,109 @@ +/* +Copyright 2019 Kohl's Department Stores, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mocks + +import ( + "testing" + + "github.com/KohlsTechnology/git2consul-go/config" + "gopkg.in/src-d/go-billy.v4/osfs" + git "gopkg.in/src-d/go-git.v4" + "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/object" + "gopkg.in/src-d/go-git.v4/storage" +) + +type Repo struct { + adds []string + Config *config.Repo + Path string + branch plumbing.ReferenceName + T *testing.T + hashes map[string]plumbing.Hash +} + +func (r *Repo) Name() string { + return "repository_mock" +} + +func (r *Repo) GetConfig() *config.Repo { + return r.Config +} + +func (r *Repo) Add(path string) { + r.adds = append(r.adds, path) +} + +func (r *Repo) CheckRef(branch string) error { + return nil +} + +func (r *Repo) CheckoutBranch(branch plumbing.ReferenceName) error { + r.branch = branch + return nil +} + +func (r *Repo) DiffStatus(commit string) (object.Changes, error) { + var changes object.Changes + for _, add := range r.adds { + changes = append(changes, &object.Change{From: object.ChangeEntry{}, To: object.ChangeEntry{Name: add}}) + } + r.adds = []string{} + return changes, nil +} + +func (r *Repo) Head() (*plumbing.Reference, error) { + if r.branch == "" { + r.branch = plumbing.NewReferenceFromStrings("master", "").Name() + r.Pull("master") + } + return plumbing.NewHashReference(r.branch, r.hashes[r.branch.Short()]), nil +} + +func (r *Repo) Pull(branch string) error { + if r.hashes == nil { + r.hashes = make(map[string]plumbing.Hash) + } + if r.hashes[branch] == plumbing.ZeroHash { + r.hashes[branch] = plumbing.ComputeHash(0, []byte(branch)) + } else { + hash := r.hashes[branch] + r.hashes[branch] = plumbing.ComputeHash(0, hash[:]) + } + r.branch = plumbing.NewReferenceFromStrings(branch, "").Name() + return nil +} + +func (r *Repo) ResolveRevision(plumbing.Revision) (*plumbing.Hash, error) { + hash := r.hashes[r.branch.Short()] + return &hash, nil +} + +func (r *Repo) Worktree() (*git.Worktree, error) { + return &git.Worktree{Filesystem: osfs.New(r.Path)}, nil +} + +func (r *Repo) Lock() {} +func (r *Repo) Unlock() {} + +func (r *Repo) GetStorer() storage.Storer { + return nil +} + +func (r *Repo) Branch() plumbing.ReferenceName { + return r.branch +} diff --git a/kv/path.go b/kv/path.go new file mode 100644 index 0000000..d7c2b19 --- /dev/null +++ b/kv/path.go @@ -0,0 +1,97 @@ +/* +Copyright 2019 Kohl's Department Stores, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kv + +import ( + "fmt" + "path" + "strings" + + "github.com/KohlsTechnology/git2consul-go/repository" +) + +//Status codes for path formatting +const ( + SourceRootNotInPrefix = iota + PathFormatterOK + PathFormatterError +) + +func getItemKey(repo repository.Repo, filePath string) (string, int, error) { + return pathHandler(repo, filePath) +} +func pathHandler(repo repository.Repo, filePath string) (string, int, error) { + filePath = strings.TrimPrefix(filePath, repository.WorkDir(repo)) + basePath, status, err := pathBaseBuilder(repo) + if err != nil { + return "", status, fmt.Errorf("Couldn't format the base of the path: %s", err) + } + corePath, status, err := pathCoreBuilder(repo, filePath) + if err != nil { + return "", status, err + } + path := path.Join(basePath, corePath) + + return path, PathFormatterOK, nil +} + +func pathBaseBuilder(repo repository.Repo) (string, int, error) { + config := repo.GetConfig() + mountPoint := config.MountPoint + key := "" + repoName := "" + if !config.SkipRepoName { + repoName = repo.Name() + } + branchName, err := getBranchName(repo) + if err != nil { + return "", PathFormatterError, err + } + if len(mountPoint) > 0 { + key = path.Join(mountPoint, repoName, branchName) + } else { + key = path.Join(repoName, branchName) + } + + return key, PathFormatterOK, nil +} + +func pathCoreBuilder(repo repository.Repo, filePath string) (string, int, error) { + config := repo.GetConfig() + sourceRoot := config.SourceRoot + + if len(sourceRoot) > 0 { + if !strings.Contains(filePath, sourceRoot) { + return "", SourceRootNotInPrefix, fmt.Errorf("Path: \"%s\" doesn't match the source_root: \"%s\"", filePath, sourceRoot) + } + filePath = strings.TrimPrefix(filePath, sourceRoot) + } + + return filePath, PathFormatterOK, nil +} + +func getBranchName(repo repository.Repo) (string, error) { + config := repo.GetConfig() + if config.SkipBranchName { + return "", nil + } + branch, err := repo.Head() + if err != nil { + return "", err + } + return branch.Name().Short(), nil +} diff --git a/kv/path_test.go b/kv/path_test.go new file mode 100644 index 0000000..6ee5578 --- /dev/null +++ b/kv/path_test.go @@ -0,0 +1,175 @@ +/* +Copyright 2019 Kohl's Department Stores, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kv + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/KohlsTechnology/git2consul-go/config" + "github.com/KohlsTechnology/git2consul-go/kv/mocks" + "github.com/KohlsTechnology/git2consul-go/repository" +) + +func TestPathHandlerWithoutMountPointAndSourceRoot(t *testing.T) { + var repo repository.Repo + repo = &mocks.Repo{Config: &config.Repo{}} + + filePath := "first_level/second_level/foo" + expectedKey := "repository_mock/master/first_level/second_level/foo" + + key, status, err := pathHandler(repo, filePath) + if err != nil { + if status == PathFormatterError { + t.Fatalf("Could not set the branch: %s", err) + } + t.Fatal(err) + } + assert.Equal(t, key, expectedKey) +} + +func TestPathHandlerWithMountPointAndSourceRoot(t *testing.T) { + var repo repository.Repo + repo = &mocks.Repo{Config: &config.Repo{}} + repo.GetConfig().MountPoint = "test_mountpoint/" + repo.GetConfig().SourceRoot = "first_level/second_level" + filePath := "first_level/second_level/foo" + expectedKey := "test_mountpoint/repository_mock/master/foo" + + key, status, err := pathHandler(repo, filePath) + if err != nil { + if status == PathFormatterError { + t.Fatalf("Could not set the branch: %s", err) + } + t.Fatal(err) + } + assert.Equal(t, key, expectedKey) +} + +func TestPathBaseBuilderWithoutMountPoint(t *testing.T) { + var repo repository.Repo + repo = &mocks.Repo{Config: &config.Repo{}} + repo.GetConfig().MountPoint = "" + expectedKey := "repository_mock/master" + key, status, err := pathBaseBuilder(repo) + if err != nil { + if status == PathFormatterError { + t.Fatalf("Could not set the branch: %s", err) + } + } + assert.Equal(t, key, expectedKey) +} + +func TestPathBaseBuilderWithMountPoint(t *testing.T) { + var repo repository.Repo + repo = &mocks.Repo{Config: &config.Repo{}} + repo.GetConfig().MountPoint = "test_mountpoint" + expectedKey := "test_mountpoint/repository_mock/master" + key, status, err := pathBaseBuilder(repo) + if err != nil { + if status == PathFormatterError { + t.Fatalf("Could not set the branch: %s", err) + } + } + assert.Equal(t, key, expectedKey) +} + +func TestPathCoreBuilderWithSourceRoot(t *testing.T) { + var repo repository.Repo + repo = &mocks.Repo{Config: &config.Repo{}} + repo.GetConfig().SourceRoot = "first_level/second_level" + defer func() { + repo.GetConfig().SourceRoot = "" + }() + expectedKey := "/foo" + filePath := "first_level/second_level/foo" + key, status, err := pathCoreBuilder(repo, filePath) + assert.NoError(t, err) + assert.Equal(t, key, expectedKey) + assert.Equal(t, status, PathFormatterOK) +} + +func TestPathCoreBuilderWithoutSourceRoot(t *testing.T) { + var repo repository.Repo + repo = &mocks.Repo{Config: &config.Repo{}} + expectedKey := "first_level/second_level/foo" + filePath := "first_level/second_level/foo" + key, status, err := pathCoreBuilder(repo, filePath) + assert.NoError(t, err) + assert.Equal(t, key, expectedKey) + assert.Equal(t, status, PathFormatterOK) +} + +func TestPathBaseBuilderSkipBranch(t *testing.T) { + var repo repository.Repo + repo = &mocks.Repo{Config: &config.Repo{}} + skipBranchName := repo.GetConfig().SkipBranchName + repo.GetConfig().SkipBranchName = true + defer func() { + repo.GetConfig().SkipBranchName = skipBranchName + }() + expectedKey := "repository_mock" + key, status, err := pathBaseBuilder(repo) + assert.NoError(t, err) + assert.Equal(t, key, expectedKey) + assert.Equal(t, status, PathFormatterOK) +} + +func TestPathBaseBuilderWithBranch(t *testing.T) { + var repo repository.Repo + repo = &mocks.Repo{Config: &config.Repo{}} + skipBranchName := repo.GetConfig().SkipBranchName + repo.GetConfig().SkipBranchName = false + defer func() { + repo.GetConfig().SkipBranchName = skipBranchName + }() + expectedKey := "repository_mock/master" + key, status, err := pathBaseBuilder(repo) + assert.NoError(t, err) + assert.Equal(t, key, expectedKey) + assert.Equal(t, status, PathFormatterOK) +} + +func TestPathBaseBuilderSkipRepo(t *testing.T) { + var repo repository.Repo + repo = &mocks.Repo{Config: &config.Repo{}} + skipRepoName := repo.GetConfig().SkipRepoName + repo.GetConfig().SkipRepoName = true + defer func() { + repo.GetConfig().SkipRepoName = skipRepoName + }() + expectedKey := "master" + key, status, err := pathBaseBuilder(repo) + assert.NoError(t, err) + assert.Equal(t, key, expectedKey) + assert.Equal(t, status, PathFormatterOK) +} + +func TestPathBaseBuilderWithRepo(t *testing.T) { + var repo repository.Repo + repo = &mocks.Repo{Config: &config.Repo{}} + skipRepoName := repo.GetConfig().SkipRepoName + repo.GetConfig().SkipRepoName = false + defer func() { + repo.GetConfig().SkipRepoName = skipRepoName + }() + expectedKey := "repository_mock/master" + key, status, err := pathBaseBuilder(repo) + assert.NoError(t, err) + assert.Equal(t, key, expectedKey) + assert.Equal(t, status, PathFormatterOK) +} diff --git a/kv/ref.go b/kv/ref.go index 626d5c9..96f69a2 100644 --- a/kv/ref.go +++ b/kv/ref.go @@ -1,15 +1,32 @@ +/* +Copyright 2019 Kohl's Department Stores, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package kv import ( "fmt" "path" - "github.com/Cimpress-MCP/go-git2consul/repository" "github.com/hashicorp/consul/api" + "github.com/KohlsTechnology/git2consul-go/repository" + "gopkg.in/src-d/go-git.v4/plumbing" ) // Get local branch ref from the KV -func (h *KVHandler) getKVRef(repo *repository.Repository, branchName string) (string, error) { +func (h *KVHandler) getKVRef(repo repository.Repo, branchName string) (string, error) { refFile := fmt.Sprintf("%s.ref", branchName) key := path.Join(repo.Name(), refFile) @@ -22,30 +39,40 @@ func (h *KVHandler) getKVRef(repo *repository.Repository, branchName string) (st if pair == nil { return "", nil } + //store the last modify index + txnItem := &api.KVTxnOp{ + Verb: api.KVCheckIndex, + Index: pair.ModifyIndex, + Key: key, + } + h.KVTxnOps = append(h.KVTxnOps, txnItem) return string(pair.Value), nil } // Put the local branch ref to the KV -func (h *KVHandler) putKVRef(repo *repository.Repository, branchName string) error { +func (h *KVHandler) putKVRef(repo repository.Repo, branchName string) error { refFile := fmt.Sprintf("%s.ref", branchName) key := path.Join(repo.Name(), refFile) - rawRef, err := repo.References.Lookup("refs/heads/" + branchName) + rawRef, err := repo.ResolveRevision(plumbing.Revision("refs/heads/" + branchName)) if err != nil { return err } - ref := rawRef.Target().String() p := &api.KVPair{ Key: key, - Value: []byte(ref), + Value: []byte(rawRef.String()), } _, err = h.Put(p, nil) if err != nil { return err } + err = h.Commit() + if err != nil { + return err + } return nil } diff --git a/kv/ref_test.go b/kv/ref_test.go new file mode 100644 index 0000000..fcd3c6a --- /dev/null +++ b/kv/ref_test.go @@ -0,0 +1,78 @@ +/* +Copyright 2019 Kohl's Department Stores, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kv + +import ( + "fmt" + "path" + "testing" + + "github.com/apex/log" + "github.com/hashicorp/consul/api" + "github.com/stretchr/testify/assert" + "github.com/KohlsTechnology/git2consul-go/config" + "github.com/KohlsTechnology/git2consul-go/kv/mocks" + "github.com/KohlsTechnology/git2consul-go/repository" +) + +//TestPutKVRef test functionality of putKVRef function. +func TestKVRef(t *testing.T) { + var repo repository.Repo + repo = &mocks.Repo{Config: &config.Repo{}, T: t} + repo.Pull("master") + handler := &KVHandler{ + API: &mocks.KV{T: t}, + logger: log.WithFields(log.Fields{ + "caller": "consul", + })} + branch, err := repo.Head() + if err != nil { + t.Fatal(err) + } + refFile := fmt.Sprintf("%s.ref", branch.Name().Short()) + key := path.Join(repo.Name(), refFile) + commit := branch.Hash().String() + + t.Run("TestPutKVRef", func(t *testing.T) { + testPutKVRef(t, branch.Name().Short(), key, commit, handler, repo) + }) + t.Run("TestPutKVRefModifiedIndex", func(t *testing.T) { + testPutKVRefModifiedIndex(t, branch.Name().Short(), key, commit, handler, repo) + }) +} + +func testPutKVRef(t *testing.T, branch string, key string, commit string, handler *KVHandler, repo repository.Repo) { + err := handler.putKVRef(repo, branch) + if err != nil { + t.Fatal(err) + } + kvBranch, _, _ := handler.Get(key, nil) + assert.Equal(t, string(kvBranch.Value), commit) + +} + +func testPutKVRefModifiedIndex(t *testing.T, branch string, key string, commit string, handler *KVHandler, repo repository.Repo) { + lastCommit, err := handler.getKVRef(repo, branch) + if err != nil { + t.Fatal(err) + } + handler.API.Put(&api.KVPair{Key: key, Value: []byte(lastCommit)}, nil) + + err = handler.putKVRef(repo, branch) + assert.IsType(t, &TransactionIntegrityError{}, err) + t.Log(err) +} diff --git a/kv/update.go b/kv/update.go index d49dd82..1c756e6 100644 --- a/kv/update.go +++ b/kv/update.go @@ -1,50 +1,93 @@ +/* +Copyright 2019 Kohl's Department Stores, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package kv import ( - "github.com/Cimpress-MCP/go-git2consul/repository" + "fmt" + "github.com/apex/log" + "github.com/KohlsTechnology/git2consul-go/repository" + "gopkg.in/src-d/go-git.v4" + "gopkg.in/src-d/go-git.v4/plumbing" ) -// HandleUpdate handles the update of a particular repository by -// comparing diffs against the KV. -func (h *KVHandler) HandleUpdate(repo *repository.Repository) error { +// HandleUpdate handles the update of a particular repository. +func (h *KVHandler) HandleUpdate(repo repository.Repo) error { + w, err := repo.Worktree() + config := repo.GetConfig() repo.Lock() defer repo.Unlock() + if err != nil { + return err + } + + for _, branch := range config.Branches { + err := w.Checkout(&git.CheckoutOptions{ + Branch: plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", branch)), + Force: true, + }) + if err != nil { + return err + } + err = h.UpdateToHead(repo) + if err != nil { + return err + } + } + return nil +} + +//UpdateToHead handles update to current HEAD comparing diffs against the KV. +func (h *KVHandler) UpdateToHead(repo repository.Repo) error { head, err := repo.Head() if err != nil { return err } - b, err := head.Branch().Name() + refName := head.Name().Short() if err != nil { return err } - h.logger.Infof("KV GET ref: %s/%s", repo.Name(), b) - kvRef, err := h.getKVRef(repo, b) + h.logger.Infof("KV GET ref: %s/%s", repo.Name(), refName) + kvRef, err := h.getKVRef(repo, refName) if err != nil { return err } // Local ref - localRef := head.Target().String() + refHash := head.Hash().String() // log.Debugf("(consul) kvRef: %s | localRef: %s", kvRef, localRef) if len(kvRef) == 0 { - log.Infof("KV PUT changes: %s/%s", repo.Name(), b) - err := h.putBranch(repo, head.Branch()) + log.Infof("KV PUT changes: %s/%s", repo.Name(), refName) + err := h.putBranch(repo, plumbing.ReferenceName(head.Name().Short())) if err != nil { return err } - err = h.putKVRef(repo, b) + err = h.putKVRef(repo, refName) if err != nil { return err } - h.logger.Infof("KV PUT ref: %s/%s", repo.Name(), b) - } else if kvRef != localRef { + h.logger.Infof("KV PUT ref: %s/%s", repo.Name(), refName) + } else if kvRef != refHash { // Check if the ref belongs to that repo - err := repo.CheckRef(kvRef) + err := repo.CheckRef(refName) if err != nil { return err } @@ -56,11 +99,11 @@ func (h *KVHandler) HandleUpdate(repo *repository.Repository) error { } h.handleDeltas(repo, deltas) - err = h.putKVRef(repo, b) + err = h.putKVRef(repo, refName) if err != nil { return err } - h.logger.Infof("KV PUT ref: %s/%s", repo.Name(), b) + h.logger.Infof("KV PUT ref: %s/%s", repo.Name(), refName) } return nil diff --git a/kv/update_test.go b/kv/update_test.go new file mode 100644 index 0000000..eca03dc --- /dev/null +++ b/kv/update_test.go @@ -0,0 +1,89 @@ +/* +Copyright 2019 Kohl's Department Stores, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kv + +import ( + "fmt" + "io/ioutil" + "os" + "path" + "path/filepath" + "strings" + "testing" + + "github.com/apex/log" + "github.com/stretchr/testify/assert" + "github.com/KohlsTechnology/git2consul-go/config" + "github.com/KohlsTechnology/git2consul-go/kv/mocks" +) + +func TestUpdateToHead(t *testing.T) { + handler := &KVHandler{ + API: &mocks.KV{T: t}, + logger: log.WithFields(log.Fields{ + "caller": "consul", + }, + ), + } + log.SetLevel(0) + //_, repoPath, _, _ := runtime.Caller(1) + repoPath, err := ioutil.TempDir("", "local-repo") + defer os.RemoveAll(repoPath) + assert.NoError(t, err) + repo := &mocks.Repo{Path: repoPath, Config: &config.Repo{}, T: t} + repo.Pull("master") + branch, err := repo.Head() + assert.NoError(t, err) + initialCommit := branch.Hash().String() + repo.Pull(branch.Name().Short()) + //Make an initial load to the Consul KV store. + handler.putBranch(repo, branch.Name()) + handler.putKVRef(repo, branch.Name().Short()) + //Fake commit + f, err := ioutil.TempFile(repoPath, "example.txt") + assert.NoError(t, err) + f.Write([]byte("A content!")) + f.Close() + fileName := strings.TrimPrefix(f.Name(), repoPath) + repo.Add(fileName) + //Pull the change. + repo.Pull(branch.Name().Short()) + + err = handler.UpdateToHead(repo) + assert.NoError(t, err) + branch, err = repo.Head() + assert.NoError(t, err) + + //Ensure ref has been updated + refFile := fmt.Sprintf("%s.ref", branch.Name().Short()) + key := path.Join(repo.Name(), refFile) + + kvBranch, _, err := handler.Get(key, nil) + + assert.Equal(t, string(kvBranch.Value), branch.Hash().String()) + + assert.NotEqual(t, string(kvBranch.Value), initialCommit) + + kvPath := filepath.Join(repo.Name(), branch.Name().Short(), fileName) + kvContent, _, err := handler.Get(kvPath, nil) + fileContent, err := ioutil.ReadFile(filepath.Join(repoPath, fileName)) + + assert.Equal(t, kvContent.Value, fileContent) + + // return nil + // }) +} diff --git a/main.go b/main.go index db946d7..dda0d73 100644 --- a/main.go +++ b/main.go @@ -1,3 +1,19 @@ +/* +Copyright 2019 Kohl's Department Stores, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package main import ( @@ -7,10 +23,10 @@ import ( "os/signal" "syscall" - "github.com/Cimpress-MCP/go-git2consul/config" - "github.com/Cimpress-MCP/go-git2consul/runner" "github.com/apex/log" "github.com/apex/log/handlers/text" + "github.com/KohlsTechnology/git2consul-go/config" + "github.com/KohlsTechnology/git2consul-go/runner" ) // Exit code represented as int values for particular errors. @@ -39,8 +55,7 @@ func main() { } if version { - fmt.Println("git2consul:") - fmt.Printf(" %-9s%s\n", "Version:", Version) + fmt.Println("git2consul", "version", Version) if GitCommit != "" { fmt.Printf(" %-9s%s\n", "Build:", GitCommit) } diff --git a/repository/auth.go b/repository/auth.go new file mode 100644 index 0000000..2e52efd --- /dev/null +++ b/repository/auth.go @@ -0,0 +1,48 @@ +/* +Copyright 2019 Kohl's Department Stores, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package repository + +import ( + "github.com/KohlsTechnology/git2consul-go/config" + "gopkg.in/src-d/go-git.v4/plumbing/transport" + "gopkg.in/src-d/go-git.v4/plumbing/transport/http" + "gopkg.in/src-d/go-git.v4/plumbing/transport/ssh" +) + +// GetAuth returns AuthMethod based on the passed flags +func GetAuth(repo *config.Repo) (transport.AuthMethod, error) { + var auth transport.AuthMethod + var err error + auth = nil + + if len(repo.Credentials.Password) > 0 && len(repo.Credentials.Username) > 0 { + auth = &http.BasicAuth{ + Username: repo.Credentials.Username, + Password: repo.Credentials.Password, + } + } else if len(repo.Credentials.PrivateKey.Key) > 0 { + if len(repo.Credentials.Username) == 0 { + repo.Credentials.Username = "git" + } + auth, err = ssh.NewPublicKeysFromFile(repo.Credentials.PrivateKey.Username, repo.Credentials.PrivateKey.Key, repo.Credentials.PrivateKey.Password) + if err != nil { + return nil, err + } + } + + return auth, err +} diff --git a/repository/auth_test.go b/repository/auth_test.go new file mode 100644 index 0000000..a250d42 --- /dev/null +++ b/repository/auth_test.go @@ -0,0 +1,84 @@ +/* +Copyright 2019 Kohl's Department Stores, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package repository + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/KohlsTechnology/git2consul-go/config/mock" + "github.com/KohlsTechnology/git2consul-go/repository/mocks" + "gopkg.in/src-d/go-git.v4/plumbing/transport/http" + "gopkg.in/src-d/go-git.v4/plumbing/transport/ssh" +) + +func TestGetAuthWithPlainAuth(t *testing.T) { + _, remotePath := mocks.InitRemote(t) + defer os.RemoveAll(remotePath) + + cfg := mock.Config(remotePath) + defer os.RemoveAll(cfg.LocalStore) + + repoConfig := cfg.Repos[0] + repoConfig.Credentials.Username = "foo" + repoConfig.Credentials.Password = "bar" + + expectedAuth := &http.BasicAuth{ + Username: "foo", + Password: "bar", + } + + auth, err := GetAuth(repoConfig) + assert.NoError(t, err) + assert.Equal(t, expectedAuth, auth) +} + +func TestGetAuthWithKeyAuth(t *testing.T) { + wd, _ := os.Getwd() + key := filepath.Join(wd, "../config/test-fixtures", "test_ssh_key") + _, remotePath := mocks.InitRemote(t) + defer os.RemoveAll(remotePath) + cfg := mock.Config(remotePath) + defer os.RemoveAll(cfg.LocalStore) + repoConfig := cfg.Repos[0] + repoConfig.Credentials.PrivateKey.Key = key + repoConfig.Credentials.PrivateKey.Username = "foo" + + expectedAuth := &ssh.PublicKeys{ + User: "foo", + Signer: nil, + } + + auth, err := GetAuth(repoConfig) + assert.NoError(t, err) + + assert.Equal(t, expectedAuth.String(), auth.String()) +} + +func TestGetAuthWithoutCred(t *testing.T) { + _, remotePath := mocks.InitRemote(t) + defer os.RemoveAll(remotePath) + cfg := mock.Config(remotePath) + defer os.RemoveAll(cfg.LocalStore) + repoConfig := cfg.Repos[0] + + auth, err := GetAuth(repoConfig) + assert.NoError(t, err) + assert.Nil(t, auth) +} diff --git a/repository/checkout.go b/repository/checkout.go index c9d90c2..cf16e78 100644 --- a/repository/checkout.go +++ b/repository/checkout.go @@ -1,83 +1,117 @@ +/* +Copyright 2019 Kohl's Department Stores, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package repository import ( + "fmt" "path" - "gopkg.in/libgit2/git2go.v24" + "gopkg.in/src-d/go-git.v4" + "gopkg.in/src-d/go-git.v4/config" + "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/storer" ) -func stringInSlice(a string, list []string) bool { - for _, b := range list { - if b == a { - return true - } - } - return false -} - -// Checkout branches specified in the config func (r *Repository) checkoutConfigBranches() error { - itr, err := r.NewBranchIterator(git.BranchRemote) + err := r.Fetch(&git.FetchOptions{ + RefSpecs: []config.RefSpec{"refs/*:refs/*", "HEAD:refs/heads/HEAD"}, + Auth: r.Authentication, + }) + + w, err := r.Worktree() + if err != nil { return err } - defer itr.Free() - - var checkoutBranchFn = func(b *git.Branch, _ git.BranchType) error { - bn, err := b.Name() - if err != nil { - return err - } - - // Only checkout tracked branches - // TODO: optimize this O(n^2) - if stringInSlice(path.Base(bn), r.Config.Branches) == false { - return nil - } - _, err = r.References.Lookup("refs/heads/" + path.Base(bn)) - if err != nil { - localRef, err := r.References.Create("refs/heads/"+path.Base(bn), b.Reference.Target(), true, "") - if err != nil { - return err - } - - err = r.SetHead(localRef.Name()) - if err != nil { - return err - } + refIter, err := remoteBranches(r.Storer) - err = r.CheckoutHead(&git.CheckoutOpts{ - Strategy: git.CheckoutForce, + _ = refIter.ForEach(func(b *plumbing.Reference) error { + branchOnRemote := StringInSlice(path.Base(b.Name().String()), r.Config.Branches) + if branchOnRemote != false { + err := w.Checkout(&git.CheckoutOptions{ + Branch: plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", b.Name())), + Force: true, }) if err != nil { return err } } - return nil + }) + + return nil +} + +//CheckoutBranch performs a checkout on the specific branch +func (r *Repository) CheckoutBranch(branch plumbing.ReferenceName) error { + err := r.Fetch(&git.FetchOptions{ + RefSpecs: []config.RefSpec{"refs/*:refs/*", "HEAD:refs/heads/HEAD"}, + Auth: r.Authentication, + Force: true, + }) + + w, err := r.Worktree() + + if err != nil { + return err } - err = itr.ForEach(checkoutBranchFn) + err = w.Checkout(&git.CheckoutOptions{ + Branch: branch, + Force: true, + }) - if err != nil && !git.IsErrorCode(err, git.ErrIterOver) { + if err != nil { return err } return nil } -// CheckoutBranch performs a checkout on a specific branch -func (r *Repository) CheckoutBranch(branch *git.Branch, opts *git.CheckoutOpts) error { - err := r.SetHead(branch.Reference.Name()) +func remoteBranches(s storer.ReferenceStorer) (storer.ReferenceIter, error) { + refs, err := s.IterReferences() if err != nil { - return err + return nil, err } - err = r.CheckoutHead(opts) + return storer.NewReferenceFilteredIter(func(ref *plumbing.Reference) bool { + return ref.Name().IsRemote() + }, refs), nil +} + +//LocalBranches returns an iterator to iterate only over local branches. +func LocalBranches(s storer.ReferenceStorer) (storer.ReferenceIter, error) { + refs, err := s.IterReferences() if err != nil { - return err + return nil, err } - return nil + return storer.NewReferenceFilteredIter(func(ref *plumbing.Reference) bool { + return !ref.Name().IsRemote() + }, refs), nil +} + +//StringInSlice checks if value exists within slice. +func StringInSlice(a string, list []string) bool { + for _, b := range list { + if b == a { + return true + } + } + return false } diff --git a/repository/checkout_test.go b/repository/checkout_test.go index 7a9a447..3c71335 100644 --- a/repository/checkout_test.go +++ b/repository/checkout_test.go @@ -1,47 +1,52 @@ +/* +Copyright 2019 Kohl's Department Stores, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package repository import ( + "io/ioutil" "os" - "path/filepath" "testing" - "github.com/Cimpress-MCP/go-git2consul/config/mock" - "github.com/Cimpress-MCP/go-git2consul/testutil" - "gopkg.in/libgit2/git2go.v24" + "github.com/stretchr/testify/assert" + "github.com/KohlsTechnology/git2consul-go/config/mock" + "github.com/KohlsTechnology/git2consul-go/repository/mocks" + git "gopkg.in/src-d/go-git.v4" ) func TestCheckoutBranch(t *testing.T) { - gitRepo, cleanup := testutil.GitInitTestRepo(t) - defer cleanup() + _, remotePath := mocks.InitRemote(t) + defer os.RemoveAll(remotePath) - repoConfig := mock.RepoConfig(gitRepo.Workdir()) - dstPath := filepath.Join(os.TempDir(), repoConfig.Name) + repoConfig := mock.RepoConfig(remotePath) - localRepo, err := git.Clone(repoConfig.Url, dstPath, &git.CloneOptions{}) - if err != nil { - t.Fatal(err) - } + dstPath, err := ioutil.TempDir("", repoConfig.Name) + assert.Nil(t, err) + defer os.RemoveAll(dstPath) + + localRepo, err := git.PlainClone(dstPath, false, &git.CloneOptions{URL: repoConfig.URL}) + assert.Nil(t, err) repo := &Repository{ Repository: localRepo, Config: repoConfig, } - branch, err := repo.LookupBranch("master", git.BranchLocal) - if err != nil { - t.Fatal(err) - } - - err = repo.CheckoutBranch(branch, &git.CheckoutOpts{}) - if err != nil { - t.Fatal(err) - } + branch := repo.Branch() - // Cleanup - defer func() { - err = os.RemoveAll(repo.Workdir()) - if err != nil { - t.Fatal(err) - } - }() + err = repo.CheckoutBranch(branch) + assert.Nil(t, err) } diff --git a/repository/clone.go b/repository/clone.go index 1162480..9a20c32 100644 --- a/repository/clone.go +++ b/repository/clone.go @@ -1,9 +1,25 @@ +/* +Copyright 2019 Kohl's Department Stores, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package repository import ( "fmt" - "gopkg.in/libgit2/git2go.v24" + "gopkg.in/src-d/go-git.v4" ) // Clone the repository. Cloning will only checkout tracked branches. @@ -12,18 +28,15 @@ func (r *Repository) Clone(path string) error { r.Lock() defer r.Unlock() - // Clone the first tracked branch instead of the default branch if len(r.Config.Branches) == 0 { return fmt.Errorf("No tracked branches specified") } - checkoutBranch := r.Config.Branches[0] - rawRepo, err := git.Clone(r.Config.Url, path, &git.CloneOptions{ - CheckoutOpts: &git.CheckoutOpts{ - Strategy: git.CheckoutNone, - }, - CheckoutBranch: checkoutBranch, + rawRepo, err := git.PlainClone(path, false, &git.CloneOptions{ + URL: r.Config.URL, + Auth: r.Authentication, }) + if err != nil { return err } diff --git a/repository/clone_test.go b/repository/clone_test.go index e31570d..a573984 100644 --- a/repository/clone_test.go +++ b/repository/clone_test.go @@ -1,35 +1,46 @@ +/* +Copyright 2019 Kohl's Department Stores, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package repository import ( + "io/ioutil" "os" - "path" "testing" - "github.com/Cimpress-MCP/go-git2consul/config/mock" - "github.com/Cimpress-MCP/go-git2consul/testutil" + "github.com/stretchr/testify/assert" + "github.com/KohlsTechnology/git2consul-go/config/mock" + "github.com/KohlsTechnology/git2consul-go/repository/mocks" ) func TestClone(t *testing.T) { - gitRepo, cleanup := testutil.GitInitTestRepo(t) - defer cleanup() + _, remotePath := mocks.InitRemote(t) + defer os.RemoveAll(remotePath) - cfg := mock.Config(gitRepo.Workdir()) + cfg := mock.Config(remotePath) + defer os.RemoveAll(cfg.LocalStore) repo := &Repository{ Config: cfg.Repos[0], } - repoPath := path.Join(cfg.LocalStore, repo.Config.Name) - err := repo.Clone(repoPath) - if err != nil { - t.Fatal(err) - } + localPath, err := ioutil.TempDir(cfg.LocalStore, repo.Config.Name) + assert.Nil(t, err) + defer os.RemoveAll(localPath) - //Cleanup cloned repo - defer func() { - err = os.RemoveAll(repo.Workdir()) - if err != nil { - t.Fatal(err) - } - }() + err = repo.Clone(localPath) + assert.Nil(t, err) } diff --git a/repository/diff.go b/repository/diff.go index 6d2b451..a2c108c 100644 --- a/repository/diff.go +++ b/repository/diff.go @@ -1,74 +1,77 @@ -package repository +/* +Copyright 2019 Kohl's Department Stores, Inc. -import "gopkg.in/libgit2/git2go.v24" +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at -// DiffStatus compares the current workdir with a target ref and return the modified files -func (r *Repository) DiffStatus(ref string) ([]git.DiffDelta, error) { - deltas := []git.DiffDelta{} + http://www.apache.org/licenses/LICENSE-2.0 - oid, err := git.NewOid(ref) - if err != nil { - return nil, err - } +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ - // This can be for a different repo - obj, err := r.Lookup(oid) - if err != nil { - return nil, err - } +package repository - commit, err := obj.AsCommit() - if err != nil { - return nil, err - } +import ( + "strings" - tree, err := commit.Tree() - if err != nil { - return nil, err - } + "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/object" +) - h, err := r.Head() +// DiffStatus compares the current workdir with a target ref and return the modified files +func (r *Repository) DiffStatus(ref string) (object.Changes, error) { + sourceRoot := strings.TrimPrefix(r.GetConfig().SourceRoot, "/") + h0, err := r.Head() if err != nil { return nil, err } - - obj2, err := r.Lookup(h.Target()) + c0, err := r.CommitObject(h0.Hash()) if err != nil { return nil, err } - - commit2, err := obj2.AsCommit() + c1, err := r.CommitObject(plumbing.NewHash(ref)) if err != nil { return nil, err } - tree2, err := commit2.Tree() - if err != nil { - return nil, err + commits := []*object.Commit{c0, c1} + if len(commits[0].ParentHashes) != 0 { + commits = []*object.Commit{c1, c0} } - do, err := git.DefaultDiffOptions() + t0, err := r.TreeObject(commits[0].TreeHash) if err != nil { return nil, err } - - diffs, err := r.DiffTreeToTree(tree, tree2, &do) + t1, err := r.TreeObject(commits[1].TreeHash) if err != nil { return nil, err } - - n, err := diffs.NumDeltas() + diff, err := t0.Diff(t1) if err != nil { return nil, err } + return applySourceRoot(diff, sourceRoot), nil +} - for i := 0; i < n; i++ { - diff, err := diffs.GetDelta(i) - if err != nil { - return nil, err +func applySourceRoot(changes object.Changes, sourceRoot string) object.Changes { + var selected object.Changes + empty := object.ChangeEntry{} + for _, change := range changes { + name := "" + if change.From != empty { + name = change.From.Name + } else { + name = change.To.Name + } + if strings.HasPrefix(name, sourceRoot) { + selected = append(selected, change) } - deltas = append(deltas, diff) } - - return deltas, nil + return selected } diff --git a/repository/diff_test.go b/repository/diff_test.go index 35ea064..674180b 100644 --- a/repository/diff_test.go +++ b/repository/diff_test.go @@ -1,26 +1,44 @@ +/* +Copyright 2019 Kohl's Department Stores, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package repository import ( + "io/ioutil" "os" - "path/filepath" "testing" - "github.com/Cimpress-MCP/go-git2consul/config/mock" - "github.com/Cimpress-MCP/go-git2consul/testutil" - "gopkg.in/libgit2/git2go.v24" + "github.com/stretchr/testify/assert" + "github.com/KohlsTechnology/git2consul-go/config/mock" + "github.com/KohlsTechnology/git2consul-go/repository/mocks" + git "gopkg.in/src-d/go-git.v4" + "gopkg.in/src-d/go-git.v4/utils/merkletrie" ) func TestDiffStatus(t *testing.T) { - gitRepo, cleanup := testutil.GitInitTestRepo(t) - defer cleanup() + remoteRepo, remotePath := mocks.InitRemote(t) + defer os.RemoveAll(remotePath) - repoConfig := mock.RepoConfig(gitRepo.Workdir()) - dstPath := filepath.Join(os.TempDir(), repoConfig.Name) + repoConfig := mock.RepoConfig(remotePath) + dstPath, err := ioutil.TempDir("", repoConfig.Name) + defer os.RemoveAll(dstPath) + assert.Nil(t, err) - localRepo, err := git.Clone(repoConfig.Url, dstPath, &git.CloneOptions{}) - if err != nil { - t.Fatal(err) - } + localRepo, err := git.PlainClone(dstPath, false, &git.CloneOptions{URL: repoConfig.URL}) + assert.Nil(t, err) repo := &Repository{ Repository: localRepo, @@ -28,38 +46,25 @@ func TestDiffStatus(t *testing.T) { } h, err := repo.Head() - if err != nil { - t.Fatal(err) - } + assert.Nil(t, err) - oldRef := h.Target().String() + oldRef := h.Hash().String() // Push a commit to the repository - testutil.GitCommitTestRepo(t) + mocks.Add(t, remoteRepo, "tree/test.yml", []byte("foo")) + mocks.Commit(t, remoteRepo, "Add test.yml file.") - _, err = repo.Pull("master") - if err != nil { - t.Fatal(err) - } + err = repo.Pull("master") + assert.Nil(t, err) deltas, err := repo.DiffStatus(oldRef) - if err != nil { - t.Fatal(err) - } + assert.Nil(t, err) - if len(deltas) == 0 { - t.Fatal("Expected deltas from pull changes") - } + assert.Len(t, deltas, 1) - if deltas[0].Status != git.DeltaModified { - t.Fatalf("Expected DeltaModified on %s", deltas[0].OldFile.Path) - } + action, err := deltas[0].Action() + assert.Nil(t, err) + + assert.Equal(t, action, merkletrie.Insert) - // Cleanup - defer func() { - err = os.RemoveAll(repo.Workdir()) - if err != nil { - t.Fatal(err) - } - }() } diff --git a/repository/load.go b/repository/load.go index ab6c139..917b868 100644 --- a/repository/load.go +++ b/repository/load.go @@ -1,10 +1,26 @@ +/* +Copyright 2019 Kohl's Department Stores, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package repository import ( "fmt" - "github.com/Cimpress-MCP/go-git2consul/config" "github.com/apex/log" + "github.com/KohlsTechnology/git2consul-go/config" ) // LoadRepos populates Repository slice from configuration. It also @@ -17,7 +33,13 @@ func LoadRepos(cfg *config.Config) ([]*Repository, error) { // Create Repository object for each repo for _, repoConfig := range cfg.Repos { - r, state, err := New(cfg.LocalStore, repoConfig) + + auth, err := GetAuth(repoConfig) + if err != nil { + return nil, fmt.Errorf("Error getting AuthMethod: %s", err) + } + + r, state, err := New(cfg.LocalStore, repoConfig, auth) if err != nil { return nil, fmt.Errorf("Error loading %s: %s", repoConfig.Name, err) } diff --git a/repository/load_test.go b/repository/load_test.go index c27939b..983a06d 100644 --- a/repository/load_test.go +++ b/repository/load_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2019 Kohl's Department Stores, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package repository import ( @@ -6,96 +22,76 @@ import ( "path/filepath" "testing" - "github.com/Cimpress-MCP/go-git2consul/config/mock" - "github.com/Cimpress-MCP/go-git2consul/testutil" "github.com/apex/log" "github.com/apex/log/handlers/discard" - "gopkg.in/libgit2/git2go.v24" + "github.com/stretchr/testify/assert" + git "gopkg.in/src-d/go-git.v4" + "gopkg.in/src-d/go-git.v4/config" + + "github.com/KohlsTechnology/git2consul-go/config/mock" + "github.com/KohlsTechnology/git2consul-go/repository/mocks" ) func init() { log.SetHandler(discard.New()) } +//TestLoadRepos test load repos from the configuration file. func TestLoadRepos(t *testing.T) { - gitRepo, cleanup := testutil.GitInitTestRepo(t) - defer cleanup() - - cfg := mock.Config(gitRepo.Workdir()) - - repos, err := LoadRepos(cfg) - if err != nil { - t.Fatal(err) - } + _, remotePath := mocks.InitRemote(t) + defer os.RemoveAll(remotePath) - // Cleanup cloning - defer func() { - for _, repo := range repos { - os.RemoveAll(repo.Workdir()) - } - }() + cfg := mock.Config(remotePath) + defer os.RemoveAll(cfg.LocalStore) + _, err := LoadRepos(cfg) + assert.Nil(t, err) } -func TestLoadRepos_existingDir(t *testing.T) { +//TestLoadReposExistingDir tests load to exsiting repo. +func TestLoadReposExistingDir(t *testing.T) { bareDir, err := ioutil.TempDir("", "bare-dir") - if err != nil { - t.Fatal(err) - } + defer os.RemoveAll(bareDir) + assert.Nil(t, err) cfg := mock.Config(bareDir) + defer os.RemoveAll(cfg.LocalStore) _, err = LoadRepos(cfg) - if err == nil { - t.Fatal("Expected failure for existing repository") - } - // Cleanup - defer func() { - os.RemoveAll(bareDir) - }() + assert.NotNil(t, err) } -func TestLoadRepos_invalidRepo(t *testing.T) { +//TestLoadReposInvalidRepo verifies failure in case wrong url provided. +func TestLoadReposInvalidRepo(t *testing.T) { cfg := mock.Config("bogus-url") - + defer os.RemoveAll(cfg.LocalStore) _, err := LoadRepos(cfg) - if err == nil { - t.Fatal("Expected failure for invalid repository url") - } + assert.NotNil(t, err) } -func TestLoadRepos_existingRepo(t *testing.T) { - gitRepo, cleanup := testutil.GitInitTestRepo(t) - defer cleanup() +func TestLoadReposExistingRepo(t *testing.T) { + _, remotePath := mocks.InitRemote(t) + defer os.RemoveAll(remotePath) - cfg := mock.Config(gitRepo.Workdir()) + cfg := mock.Config(remotePath) + defer os.RemoveAll(cfg.LocalStore) localRepoPath := filepath.Join(cfg.LocalStore, cfg.Repos[0].Name) // Init a repo in the local store, with same name are the "remote" err := os.Mkdir(localRepoPath, 0755) - if err != nil { - t.Fatal(err) - } - - repo, err := git.InitRepository(localRepoPath, false) - if err != nil { - t.Fatal(err) - } + assert.Nil(t, err) - _, err = repo.Remotes.Create("origin", "/foo/bar") - if err != nil { - t.Fatal(err) - } + repo, err := git.PlainInit(localRepoPath, false) + assert.Nil(t, err) - repos, err := LoadRepos(cfg) + _, err = repo.CreateRemote(&config.RemoteConfig{ + Name: "origin", + URLs: []string{"/foo/bar"}, + }) if err != nil { t.Fatal(err) } - // Cleanup cloning - defer func() { - for _, repo := range repos { - os.RemoveAll(repo.Workdir()) - } - }() + _, err = LoadRepos(cfg) + assert.Nil(t, err) } diff --git a/repository/mock/mock.go b/repository/mock/mock.go deleted file mode 100644 index 61e04c5..0000000 --- a/repository/mock/mock.go +++ /dev/null @@ -1,39 +0,0 @@ -package mock - -import ( - "fmt" - "io/ioutil" - - "github.com/Cimpress-MCP/go-git2consul/config/mock" - "github.com/Cimpress-MCP/go-git2consul/repository" - - "gopkg.in/libgit2/git2go.v24" -) - -// Repository returns a mock of a repository.Repository object -func Repository(gitRepo *git.Repository) *repository.Repository { - if gitRepo == nil { - return nil - } - - repoConfig := mock.RepoConfig(gitRepo.Workdir()) - - dstPath, err := ioutil.TempDir("", "git2consul-test-local") - if err != nil { - fmt.Println(err) - return nil - } - - localRepo, err := git.Clone(repoConfig.Url, dstPath, &git.CloneOptions{}) - if err != nil { - fmt.Print(err) - return nil - } - - repo := &repository.Repository{ - Repository: localRepo, - Config: repoConfig, - } - - return repo -} diff --git a/repository/mocks/repository.go b/repository/mocks/repository.go new file mode 100644 index 0000000..2202f31 --- /dev/null +++ b/repository/mocks/repository.go @@ -0,0 +1,91 @@ +/* +Copyright 2019 Kohl's Department Stores, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mocks + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + "time" + + git "gopkg.in/src-d/go-git.v4" + "gopkg.in/src-d/go-git.v4/plumbing/object" +) + +func InitRemote(t *testing.T) (*git.Repository, string) { + repoPath, err := ioutil.TempDir("", "git2consul-remote") + if err != nil { + t.Fatal(err) + } + + t.Logf("Initializing new repo in %s", repoPath) + repo, err := git.PlainInit(repoPath, false) + if err != nil { + t.Fatal(err) + } + + Add(t, repo, "example/foo.txt", []byte("Example content foo.txt")) + Add(t, repo, "example/boo.txt", []byte("Example content boo.txt")) + Commit(t, repo, "Initial commit") + return repo, repoPath +} + +func Add(t *testing.T, repo *git.Repository, path string, content []byte) { + w, err := repo.Worktree() + if err != nil { + t.Fatal(err) + } + root := w.Filesystem.Root() + dir := filepath.Dir(path) + fileName := filepath.Base(path) + _, err = os.Stat(filepath.Join(root, dir)) + if os.IsNotExist(err) { + err := os.Mkdir(filepath.Join(root, dir), 0700) + if err != nil { + t.Fatal(err) + } + } + f, err := os.Create(filepath.Join(root, dir, fileName)) + if err != nil { + t.Fatal(err) + } + defer f.Close() + _, err = f.Write(content) + if err != nil { + t.Fatal(err) + } + w.Add(path) +} + +func getSignature() *object.Signature { + when := time.Now() + sig := &object.Signature{ + Name: "foo", + Email: "foo@foo.foo", + When: when, + } + return sig +} + +func Commit(t *testing.T, repo *git.Repository, message string) { + w, err := repo.Worktree() + if err != nil { + t.Fatal(err) + } + w.Commit(message, &git.CommitOptions{Author: getSignature()}) +} diff --git a/repository/pull.go b/repository/pull.go index 2a045e3..d47e895 100644 --- a/repository/pull.go +++ b/repository/pull.go @@ -1,91 +1,58 @@ +/* +Copyright 2019 Kohl's Department Stores, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package repository import ( "fmt" - "gopkg.in/libgit2/git2go.v24" + "gopkg.in/src-d/go-git.v4" + "gopkg.in/src-d/go-git.v4/plumbing" ) // Pull a repository branch, which is equivalent to a fetch and merge -func (r *Repository) Pull(branchName string) (git.MergeAnalysis, error) { +func (r *Repository) Pull(branchName string) error { r.Lock() defer r.Unlock() - origin, err := r.Remotes.Lookup("origin") - if err != nil { - return 0, err - } - defer origin.Free() - - rawLocalBranchRef := fmt.Sprintf("refs/heads/%s", branchName) + w, err := r.Worktree() - // Fetch - err = origin.Fetch([]string{rawLocalBranchRef}, nil, "") if err != nil { - return 0, err + return err } - rawRemoteBranchRef := fmt.Sprintf("refs/remotes/origin/%s", branchName) - - remoteBranchRef, err := r.References.Lookup(rawRemoteBranchRef) - if err != nil { - return 0, err - } - - // If the ref on the branch doesn't exist locally, create it - // This also creates the branch - _, err = r.References.Lookup(rawLocalBranchRef) - if err != nil { - _, err = r.References.Create(rawLocalBranchRef, remoteBranchRef.Target(), true, "") - if err != nil { - return 0, err - } - } - - // Change the HEAD to current branch and checkout - err = r.SetHead(rawLocalBranchRef) - if err != nil { - return 0, err - } - err = r.CheckoutHead(&git.CheckoutOpts{ - Strategy: git.CheckoutForce, + err = w.Checkout(&git.CheckoutOptions{ + Branch: plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", branchName)), + Force: true, }) - if err != nil { - return 0, err - } - - head, err := r.Head() - if err != nil { - return 0, err - } - // Create annotated commit - annotatedCommit, err := r.AnnotatedCommitFromRef(remoteBranchRef) if err != nil { - return 0, err + return err } - // Merge analysis - mergeHeads := []*git.AnnotatedCommit{annotatedCommit} - analysis, _, err := r.MergeAnalysis(mergeHeads) + err = w.Pull(&git.PullOptions{ + RemoteName: "origin", + ReferenceName: plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", branchName)), + Auth: r.Authentication, + SingleBranch: true, + Force: true, + }) if err != nil { - return 0, err - } - - // Action on analysis - switch { - case analysis&git.MergeAnalysisFastForward != 0, analysis&git.MergeAnalysisNormal != 0: - if err := r.Merge(mergeHeads, nil, nil); err != nil { - return 0, err - } - - // Update refs on heads (local) from remotes - if _, err = head.SetTarget(remoteBranchRef.Target(), ""); err != nil { - return analysis, err - } + return err } - defer head.Free() - defer r.StateCleanup() - return analysis, nil + return nil } diff --git a/repository/pull_test.go b/repository/pull_test.go index 547aa39..481b327 100644 --- a/repository/pull_test.go +++ b/repository/pull_test.go @@ -1,26 +1,44 @@ +/* +Copyright 2019 Kohl's Department Stores, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package repository import ( + "io/ioutil" "os" "path/filepath" "testing" - "github.com/Cimpress-MCP/go-git2consul/config/mock" - "github.com/Cimpress-MCP/go-git2consul/testutil" - "gopkg.in/libgit2/git2go.v24" + "github.com/stretchr/testify/assert" + "github.com/KohlsTechnology/git2consul-go/config/mock" + "github.com/KohlsTechnology/git2consul-go/repository/mocks" + git "gopkg.in/src-d/go-git.v4" ) func TestPull(t *testing.T) { - gitRepo, cleanup := testutil.GitInitTestRepo(t) - defer cleanup() + remoteRepo, remotePath := mocks.InitRemote(t) + defer os.RemoveAll(remotePath) - repoConfig := mock.RepoConfig(gitRepo.Workdir()) - dstPath := filepath.Join(os.TempDir(), repoConfig.Name) + repoConfig := mock.RepoConfig(remotePath) + dstPath, err := ioutil.TempDir("", repoConfig.Name) + defer os.RemoveAll(dstPath) + assert.Nil(t, err) - localRepo, err := git.Clone(repoConfig.Url, dstPath, &git.CloneOptions{}) - if err != nil { - t.Fatal(err) - } + localRepo, err := git.PlainClone(dstPath, false, &git.CloneOptions{URL: repoConfig.URL}) + assert.Nil(t, err) repo := &Repository{ Repository: localRepo, @@ -28,18 +46,12 @@ func TestPull(t *testing.T) { } // Push a commit to the repository - testutil.GitCommitTestRepo(t) + mocks.Add(t, remoteRepo, "tree/test.yml", []byte("foo")) + mocks.Commit(t, remoteRepo, "Add test.yml file.") + err = repo.Pull("master") + assert.Nil(t, err) - _, err = repo.Pull("master") - if err != nil { - t.Fatal(err) - } + _, err = os.Stat(filepath.Join(dstPath, "tree/test.yml")) + assert.Nil(t, err) - // Cleanup - defer func() { - err = os.RemoveAll(repo.Workdir()) - if err != nil { - t.Fatal(err) - } - }() } diff --git a/repository/ref.go b/repository/ref.go index 93bcbc0..f0ace93 100644 --- a/repository/ref.go +++ b/repository/ref.go @@ -1,16 +1,28 @@ +/* +Copyright 2019 Kohl's Department Stores, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package repository -import "gopkg.in/libgit2/git2go.v24" +import ( + "gopkg.in/src-d/go-git.v4/plumbing" +) // CheckRef checks whether a particular ref is part of the repository func (r *Repository) CheckRef(ref string) error { - oid, err := git.NewOid(ref) - if err != nil { - return err - } - - // This can be for a different repo - _, err = r.Lookup(oid) + _, err := r.ResolveRevision(plumbing.Revision(ref)) if err != nil { return err } diff --git a/repository/ref_test.go b/repository/ref_test.go index 51753ec..6806099 100644 --- a/repository/ref_test.go +++ b/repository/ref_test.go @@ -1,26 +1,43 @@ +/* +Copyright 2019 Kohl's Department Stores, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package repository import ( + "io/ioutil" "os" - "path/filepath" "testing" - "github.com/Cimpress-MCP/go-git2consul/config/mock" - "github.com/Cimpress-MCP/go-git2consul/testutil" - "gopkg.in/libgit2/git2go.v24" + "github.com/stretchr/testify/assert" + "github.com/KohlsTechnology/git2consul-go/config/mock" + "github.com/KohlsTechnology/git2consul-go/repository/mocks" + git "gopkg.in/src-d/go-git.v4" ) func TestCheckRef(t *testing.T) { - gitRepo, cleanup := testutil.GitInitTestRepo(t) - defer cleanup() + remoteRepo, remotePath := mocks.InitRemote(t) + defer os.RemoveAll(remotePath) - repoConfig := mock.RepoConfig(gitRepo.Workdir()) - dstPath := filepath.Join(os.TempDir(), repoConfig.Name) + repoConfig := mock.RepoConfig(remotePath) + dstPath, err := ioutil.TempDir("", repoConfig.Name) + assert.Nil(t, err) + defer os.RemoveAll(dstPath) - localRepo, err := git.Clone(repoConfig.Url, dstPath, &git.CloneOptions{}) - if err != nil { - t.Fatal(err) - } + localRepo, err := git.PlainClone(dstPath, false, &git.CloneOptions{URL: repoConfig.URL}) + assert.Nil(t, err) repo := &Repository{ Repository: localRepo, @@ -28,30 +45,17 @@ func TestCheckRef(t *testing.T) { } h, err := repo.Head() - if err != nil { - t.Fatal(err) - } - - ref := h.Target().String() + assert.Nil(t, err) + ref := h.Name().Short() // Push a commit to the repository - testutil.GitCommitTestRepo(t) + mocks.Add(t, remoteRepo, "tree/test.yml", []byte("foo")) + mocks.Commit(t, remoteRepo, "Add test.yml file.") - _, err = repo.Pull("master") - if err != nil { - t.Fatal(err) - } + err = repo.Pull("master") + assert.Nil(t, err) err = repo.CheckRef(ref) - if err != nil { - t.Fatal(err) - } + assert.Nil(t, err) - // Cleanup - defer func() { - err = os.RemoveAll(repo.Workdir()) - if err != nil { - t.Fatal(err) - } - }() } diff --git a/repository/repository.go b/repository/repository.go index f9d3467..873832a 100644 --- a/repository/repository.go +++ b/repository/repository.go @@ -1,22 +1,64 @@ +/* +Copyright 2019 Kohl's Department Stores, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package repository import ( "fmt" "os" "path/filepath" - "strings" "sync" - "github.com/Cimpress-MCP/go-git2consul/config" - "gopkg.in/libgit2/git2go.v24" + "gopkg.in/src-d/go-git.v4/plumbing/object" + "gopkg.in/src-d/go-git.v4/storage" + + "github.com/KohlsTechnology/git2consul-go/config" + "gopkg.in/src-d/go-git.v4" + "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/transport" ) +const ( + refHead = "refs/heads" +) + +// Repo interface represents Repository +type Repo interface { + Name() string + Pull(string) error + CheckoutBranch(plumbing.ReferenceName) error + CheckRef(string) error + Head() (*plumbing.Reference, error) + Lock() + Unlock() + DiffStatus(string) (object.Changes, error) + Worktree() (*git.Worktree, error) + Branch() plumbing.ReferenceName + GetConfig() *config.Repo + GetStorer() storage.Storer + ResolveRevision(plumbing.Revision) (*plumbing.Hash, error) +} + // Repository is used to hold the git repository object and it's configuration type Repository struct { sync.Mutex *git.Repository - Config *config.Repo + Config *config.Repo + Authentication transport.AuthMethod } // Status codes for Repository object creation @@ -26,18 +68,28 @@ const ( RepositoryOpened ) +// GetConfig returns config *Repo +func (r *Repository) GetConfig() *config.Repo { + return r.Config +} + +// GetStorer returns Storer +func (r *Repository) GetStorer() storage.Storer { + return r.Storer +} + // Name returns the repository name func (r *Repository) Name() string { return r.Config.Name } // Branch returns the branch name -func (r *Repository) Branch() string { +func (r *Repository) Branch() plumbing.ReferenceName { head, err := r.Head() if err != nil { return "" } - bn, err := head.Branch().Name() + bn := head.Name() if err != nil { return "" } @@ -46,12 +98,13 @@ func (r *Repository) Branch() string { } // New is used to construct a new repository object from the configuration -func New(repoBasePath string, repoConfig *config.Repo) (*Repository, int, error) { +func New(repoBasePath string, repoConfig *config.Repo, auth transport.AuthMethod) (*Repository, int, error) { repoPath := filepath.Join(repoBasePath, repoConfig.Name) r := &Repository{ - Repository: &git.Repository{}, - Config: repoConfig, + Repository: &git.Repository{}, + Config: repoConfig, + Authentication: auth, } state, err := r.init(repoPath) @@ -66,43 +119,37 @@ func New(repoBasePath string, repoConfig *config.Repo) (*Repository, int, error) return r, state, nil } -// Initialize git.Repository object by opening the repostory or cloning from +// Initialize git.Repository object by opening the repostiry or cloning from // the source URL. It does not handle purging existing file or directory // with the same path func (r *Repository) init(repoPath string) (int, error) { - gitRepo, err := git.OpenRepository(repoPath) + gitRepo, err := git.PlainOpen(repoPath) if err != nil || gitRepo == nil { err := r.Clone(repoPath) if err != nil { - return RepositoryError, err - } - return RepositoryCloned, nil - } - - // If remote URL are not the same, it will purge local copy and re-clone - if r.mismatchRemoteUrl(gitRepo) { - os.RemoveAll(gitRepo.Workdir()) - err := r.Clone(repoPath) - if err != nil { - return RepositoryError, err + // more explicit error handling as a workaround for the upstream issue, tracked under: + // https://github.com/src-d/go-git/issues/741 + switch err { + case transport.ErrAuthenticationRequired: + os.RemoveAll(repoPath) + return RepositoryError, err + case transport.ErrAuthorizationFailed: + os.RemoveAll(repoPath) + return RepositoryError, err + default: + os.Remove(repoPath) + return RepositoryError, err + } } return RepositoryCloned, nil } r.Repository = gitRepo - return RepositoryOpened, nil } -func (r *Repository) mismatchRemoteUrl(gitRepo *git.Repository) bool { - rm, err := gitRepo.Remotes.Lookup("origin") - if err != nil { - return true - } - - if strings.Compare(rm.Url(), r.Config.Url) != 0 { - return true - } - - return false +//WorkDir returns working directory for a local copy of the repository. +func WorkDir(r Repo) string { + w, _ := r.Worktree() + return w.Filesystem.Root() } diff --git a/repository/repository_test.go b/repository/repository_test.go index 0cc73ad..43a6a11 100644 --- a/repository/repository_test.go +++ b/repository/repository_test.go @@ -1,44 +1,47 @@ +/* +Copyright 2019 Kohl's Department Stores, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package repository import ( "os" "testing" - "github.com/Cimpress-MCP/go-git2consul/config/mock" - "github.com/Cimpress-MCP/go-git2consul/testutil" + "github.com/stretchr/testify/assert" + + "github.com/KohlsTechnology/git2consul-go/config/mock" + "github.com/KohlsTechnology/git2consul-go/repository/mocks" ) +//TestNew verifies repository.Repository object iniciator. func TestNew(t *testing.T) { - gitRepo, cleanup := testutil.GitInitTestRepo(t) - defer cleanup() - - cfg := mock.Config(gitRepo.Workdir()) + _, remotePath := mocks.InitRemote(t) + defer os.RemoveAll(remotePath) + cfg := mock.Config(remotePath) + defer os.RemoveAll(cfg.LocalStore) repoConfig := cfg.Repos[0] - repo, status, err := New(cfg.LocalStore, repoConfig) - if err != nil { - t.Fatal(err) - } + _, status, err := New(cfg.LocalStore, repoConfig, nil) + assert.Nil(t, err) - if status != RepositoryCloned { - t.Fatalf("Expected clone status") - } + assert.Equal(t, status, RepositoryCloned) // Call New() again, this time expecting RepositoryOpened - repo, status, err = New(cfg.LocalStore, repoConfig) - if err != nil { - t.Fatal(err) - } - - if status != RepositoryOpened { - t.Fatalf("Expected clone status") - } - - // Cleanup cloning - defer func() { - err := os.RemoveAll(repo.Workdir()) - if err != nil { - t.Fatal(err) - } - }() + _, status, err = New(cfg.LocalStore, repoConfig, nil) + assert.Nil(t, err) + assert.Equal(t, status, RepositoryOpened) + } diff --git a/runner/runner.go b/runner/runner.go index 0c324ce..08343b8 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1,13 +1,30 @@ +/* +Copyright 2019 Kohl's Department Stores, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package runner import ( "fmt" + "time" - "github.com/Cimpress-MCP/go-git2consul/config" - "github.com/Cimpress-MCP/go-git2consul/kv" - "github.com/Cimpress-MCP/go-git2consul/repository" - "github.com/Cimpress-MCP/go-git2consul/watcher" "github.com/apex/log" + "github.com/KohlsTechnology/git2consul-go/config" + "github.com/KohlsTechnology/git2consul-go/kv" + "github.com/KohlsTechnology/git2consul-go/repository" + "github.com/KohlsTechnology/git2consul-go/watcher" ) // Runner is used to initialize a watcher and kvHandler @@ -23,13 +40,14 @@ type Runner struct { once bool - kvHandler *kv.KVHandler + kvHandler kv.Handler watcher *watch.Watcher } // NewRunner creates a new runner instance func NewRunner(config *config.Config, once bool) (*Runner, error) { + // var repos repository.Repo logger := log.WithField("caller", "runner") // Create repos from configuration @@ -37,9 +55,12 @@ func NewRunner(config *config.Config, once bool) (*Runner, error) { if err != nil { return nil, fmt.Errorf("Cannot load repositories from configuration: %s", err) } - + var reposI = make([]repository.Repo, len(repos)) + for index, repo := range repos { + reposI[index] = repo + } // Create watcher to watch for repo changes - watcher := watch.New(repos, config.HookSvr, once) + watcher := watch.New(reposI, config.HookSvr, once) // Create the handler handler, err := kv.New(config.Consul) @@ -70,10 +91,15 @@ func (r *Runner) Start() { select { case repo := <-r.watcher.RepoChangeCh: // Handle change, and return if error on handler - err := r.kvHandler.HandleUpdate(repo) + retry := 0 + var err error + for ok := true; ok && retry < 3; retry++ { + err = r.kvHandler.HandleUpdate(repo) + _, ok = err.(*kv.TransactionIntegrityError) + time.Sleep(1000 * time.Millisecond) + } if err != nil { r.ErrCh <- err - return } case <-r.watcher.SndDoneCh: // This triggers when watcher gets an error that causes termination r.logger.Info("Watcher received finish") diff --git a/testutil/git_repo.go b/testutil/git_repo.go deleted file mode 100644 index 2d8a67b..0000000 --- a/testutil/git_repo.go +++ /dev/null @@ -1,204 +0,0 @@ -// Package testutil takes care of initializing a local git repository for -// testing. The 'remote' should match the one specified in config/mock. -package testutil - -import ( - "io" - "io/ioutil" - "os" - "path" - "path/filepath" - "runtime" - "strings" - "testing" - "time" - - "gopkg.in/libgit2/git2go.v24" -) - -var testRepo *git.Repository - -// Return the test-fixtures path in testutil -func fixturesRepo(t *testing.T) string { - _, filename, _, ok := runtime.Caller(0) - if !ok { - t.Fatal("Cannot find path") - } - - testutilPath := filepath.Dir(filename) - return filepath.Join(testutilPath, "test-fixtures", "example") -} - -func copyDir(srcPath string, dstPath string) error { - // Copy fixtures into temporary path. filepath is the full path - var copyFn = func(path string, info os.FileInfo, err error) error { - currentFilePath := strings.TrimPrefix(path, srcPath) - targetPath := filepath.Join(dstPath, currentFilePath) - if info.IsDir() { - if targetPath != dstPath { - err := os.Mkdir(targetPath, 0755) - if err != nil { - return err - } - } - } else { - src, err := os.Open(path) - if err != nil { - return err - } - dst, err := os.Create(targetPath) - if err != nil { - return err - } - - _, err = io.Copy(dst, src) - if err != nil { - return err - } - } - - return nil - } - - err := filepath.Walk(srcPath, copyFn) - return err -} - -// GitInitTestRepo coppies test-fixtures to os.TempDir() and performs a -// git-init on directory. -func GitInitTestRepo(t *testing.T) (*git.Repository, func()) { - fixtureRepo := fixturesRepo(t) - repoPath, err := ioutil.TempDir("", "git2consul-test-remote") - if err != nil { - t.Fatal(err) - } - - err = copyDir(fixtureRepo, repoPath) - if err != nil { - t.Fatal(err) - } - - // Init repo - repo, err := git.InitRepository(repoPath, false) - if err != nil { - t.Fatal(err) - } - - // Add files to index - idx, err := repo.Index() - if err != nil { - t.Fatal(err) - } - err = idx.AddAll([]string{}, git.IndexAddDefault, nil) - if err != nil { - t.Fatal(err) - } - err = idx.Write() - if err != nil { - t.Fatal(err) - } - - treeId, err := idx.WriteTree() - if err != nil { - t.Fatal(err) - } - - tree, err := repo.LookupTree(treeId) - if err != nil { - t.Fatal(err) - } - - // Initial commit - sig := &git.Signature{ - Name: "Test Example", - Email: "tes@example.com", - When: time.Date(2016, 01, 01, 12, 00, 00, 0, time.UTC), - } - - repo.CreateCommit("HEAD", sig, sig, "Initial commit", tree) - testRepo = repo - - // Cleanup function that removes the repository directory - var cleanup = func() { - os.RemoveAll(repoPath) - } - - return repo, cleanup -} - -// GitCommitTestRepo performs a commit on the test repository, and returns -// its Oid as well as a cleanup function to revert those changes. -func GitCommitTestRepo(t *testing.T) (*git.Oid, func()) { - // Save commmit ref for reset later - h, err := testRepo.Head() - if err != nil { - t.Fatal(err) - } - - obj, err := testRepo.Lookup(h.Target()) - if err != nil { - t.Fatal(err) - } - - initialCommit, err := obj.AsCommit() - if err != nil { - t.Fatal(err) - } - - date := []byte(time.Now().String()) - file := path.Join(testRepo.Workdir(), "foo") - err = ioutil.WriteFile(file, date, 0755) - if err != nil { - t.Fatal(err) - } - - // Commit changes - idx, err := testRepo.Index() - if err != nil { - t.Fatal(err) - } - - err = idx.AddByPath("foo") - if err != nil { - t.Fatal(err) - } - - treeId, err := idx.WriteTree() - - tree, err := testRepo.LookupTree(treeId) - if err != nil { - t.Fatal(err) - } - - h, err = testRepo.Head() - if err != nil { - t.Fatal(err) - } - - commit, err := testRepo.LookupCommit(h.Target()) - if err != nil { - t.Fatal(err) - } - - sig := &git.Signature{ - Name: "Test Example", - Email: "test@example.com", - When: time.Date(2016, 01, 01, 12, 00, 00, 0, time.UTC), - } - - oid, err := testRepo.CreateCommit("HEAD", sig, sig, "Update commit", tree, commit) - if err != nil { - t.Fatal(err) - } - - // Undo commit - var cleanup = func() { - testRepo.ResetToCommit(initialCommit, git.ResetHard, &git.CheckoutOpts{ - Strategy: git.CheckoutForce, - }) - - testRepo.StateCleanup() - } - - return oid, cleanup -} diff --git a/testutil/test-fixtures/example.json b/testutil/test-fixtures/example.json deleted file mode 100644 index 6d4df83..0000000 --- a/testutil/test-fixtures/example.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "repos": [ - { - "name": "test-example", - "url": "./test-fixtures/example", - "branches": [ - "master" - ], - "hooks": [ - { - "type": "polling", - "interval": 5 - } - ] - } - ] -} diff --git a/testutil/test-fixtures/example/foo b/testutil/test-fixtures/example/foo deleted file mode 100644 index cfd4a1c..0000000 --- a/testutil/test-fixtures/example/foo +++ /dev/null @@ -1 +0,0 @@ -This is foo's content update 69 diff --git a/testutil/test-fixtures/example/tree/bang b/testutil/test-fixtures/example/tree/bang deleted file mode 100644 index 1d4a587..0000000 --- a/testutil/test-fixtures/example/tree/bang +++ /dev/null @@ -1 +0,0 @@ -This is bang's content diff --git a/testutil/test-fixtures/example/types/bar.json b/testutil/test-fixtures/example/types/bar.json deleted file mode 100644 index 6322fa9..0000000 --- a/testutil/test-fixtures/example/types/bar.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "key_1": "value_1" -} diff --git a/version.go b/version.go index 3efdf8d..e0f79ca 100644 --- a/version.go +++ b/version.go @@ -1,3 +1,19 @@ +/* +Copyright 2019 Kohl's Department Stores, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package main // The git commit that will be used to describe the version @@ -6,4 +22,4 @@ var ( ) // Version of the program -const Version = "0.0.1" +const Version = "0.1.0" diff --git a/watcher/interval.go b/watcher/interval.go index 0bb8a8d..d8f5ba6 100644 --- a/watcher/interval.go +++ b/watcher/interval.go @@ -1,19 +1,38 @@ +/* +Copyright 2019 Kohl's Department Stores, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package watch import ( + "path" "sync" "time" - "github.com/Cimpress-MCP/go-git2consul/repository" - "gopkg.in/libgit2/git2go.v24" + "github.com/KohlsTechnology/git2consul-go/repository" + "gopkg.in/src-d/go-git.v4" + "gopkg.in/src-d/go-git.v4/plumbing" ) // Watch the repo by interval. This is called as a go routine since // ticker blocks -func (w *Watcher) pollByInterval(repo *repository.Repository, wg *sync.WaitGroup) { +func (w *Watcher) pollByInterval(repo repository.Repo, wg *sync.WaitGroup) { defer wg.Done() + config := repo.GetConfig() - hooks := repo.Config.Hooks + hooks := config.Hooks interval := time.Second // Find polling hook @@ -51,40 +70,40 @@ func (w *Watcher) pollByInterval(repo *repository.Repository, wg *sync.WaitGroup } } -// Watch all branches of a repository -func (w *Watcher) pollBranches(repo *repository.Repository) error { - itr, err := repo.NewBranchIterator(git.BranchLocal) +func (w *Watcher) pollBranches(repo repository.Repo) error { + storer := repo.GetStorer() + config := repo.GetConfig() + itr, err := repository.LocalBranches(storer) if err != nil { return err } - defer itr.Free() - - var checkoutBranchFn = func(b *git.Branch, _ git.BranchType) error { - branchName, err := b.Name() - if err != nil { - return err - } - analysis, err := repo.Pull(branchName) - if err != nil { - return err - } - - // If there is a change, send the repo RepoChangeCh - switch { - case analysis&git.MergeAnalysisUpToDate != 0: - w.logger.Debugf("Up to date: %s/%s", repo.Name(), branchName) - case analysis&git.MergeAnalysisNormal != 0, analysis&git.MergeAnalysisFastForward != 0: - w.logger.Infof("Changed: %s/%s", repo.Name(), branchName) - w.RepoChangeCh <- repo + changed := false + + var checkoutBranchFn = func(b *plumbing.Reference) error { + branchOnRemote := repository.StringInSlice(path.Base(b.Name().String()), config.Branches) + if branchOnRemote { + branchName := b.Name().Short() + err := repo.Pull(branchName) + if err == git.NoErrAlreadyUpToDate { + w.logger.Debugf("Up to date: %s/%s", repo.Name(), branchName) + } else if err != nil { + w.logger.Debugf("Unable to pull \"%s\" branch because of \"%s\"", branchName, err) + } else { + w.logger.Infof("Changed: %s/%s", repo.Name(), branchName) + changed = true + } } - return nil } err = itr.ForEach(checkoutBranchFn) - if err != nil && !git.IsErrorCode(err, git.ErrIterOver) { + if err != nil { return err } + if changed { + w.RepoChangeCh <- repo + } + return nil } diff --git a/watcher/interval_test.go b/watcher/interval_test.go index 32ae9b2..b2620f0 100644 --- a/watcher/interval_test.go +++ b/watcher/interval_test.go @@ -1,14 +1,32 @@ +/* +Copyright 2019 Kohl's Department Stores, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package watch import ( "os" + "path/filepath" "testing" - "github.com/Cimpress-MCP/go-git2consul/repository" - "github.com/Cimpress-MCP/go-git2consul/repository/mock" - "github.com/Cimpress-MCP/go-git2consul/testutil" "github.com/apex/log" "github.com/apex/log/handlers/discard" + "github.com/stretchr/testify/assert" + "github.com/KohlsTechnology/git2consul-go/config/mock" + "github.com/KohlsTechnology/git2consul-go/repository" + "github.com/KohlsTechnology/git2consul-go/repository/mocks" ) func init() { @@ -16,20 +34,22 @@ func init() { } func TestPollBranches(t *testing.T) { - gitRepo, cleanup := testutil.GitInitTestRepo(t) - defer cleanup() + remote, remotePath := mocks.InitRemote(t) + defer os.RemoveAll(remotePath) - repo := mock.Repository(gitRepo) + cfg := mock.Config(remotePath) + defer os.RemoveAll(cfg.LocalStore) + repoConfig := cfg.Repos[0] - oid, _ := testutil.GitCommitTestRepo(t) - odb, err := repo.Odb() - if err != nil { - t.Fatal(err) - } + repo, _, err := repository.New(cfg.LocalStore, repoConfig, nil) + assert.NoError(t, err) + + mocks.Add(t, remote, "example/check_interval.txt", []byte("Example content for checke_interval")) + mocks.Commit(t, remote, "Interval check") w := &Watcher{ - Repositories: []*repository.Repository{repo}, - RepoChangeCh: make(chan *repository.Repository, 1), + Repositories: []repository.Repo{repo}, + RepoChangeCh: make(chan repository.Repo, 1), ErrCh: make(chan error), RcvDoneCh: make(chan struct{}, 1), SndDoneCh: make(chan struct{}, 1), @@ -39,19 +59,7 @@ func TestPollBranches(t *testing.T) { } err = w.pollBranches(repo) - if err != nil { - t.Fatal(err) - } - - if !odb.Exists(oid) { - t.Fatal("Commit not present on remote") - } + assert.NoError(t, err) - // Cleanup on git2consul cached repo - defer func() { - err = os.RemoveAll(repo.Workdir()) - if err != nil { - t.Fatal(err) - } - }() + assert.FileExists(t, filepath.Join(repository.WorkDir(repo), "example", "check_interval.txt")) } diff --git a/watcher/watcher.go b/watcher/watcher.go index ac1d395..9438291 100644 --- a/watcher/watcher.go +++ b/watcher/watcher.go @@ -1,11 +1,27 @@ +/* +Copyright 2019 Kohl's Department Stores, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package watch import ( "sync" - "github.com/Cimpress-MCP/go-git2consul/config" - "github.com/Cimpress-MCP/go-git2consul/repository" "github.com/apex/log" + "github.com/KohlsTechnology/git2consul-go/config" + "github.com/KohlsTechnology/git2consul-go/repository" ) // Watcher is used to keep track of changes of the repositories @@ -13,9 +29,9 @@ type Watcher struct { sync.Mutex logger *log.Entry - Repositories []*repository.Repository + Repositories []repository.Repo - RepoChangeCh chan *repository.Repository + RepoChangeCh chan repository.Repo ErrCh chan error RcvDoneCh chan struct{} SndDoneCh chan struct{} @@ -24,10 +40,10 @@ type Watcher struct { once bool } -// New create a new watcher, passing in the the repositories, webhook +// New create a new watcher, passing in the repositories, webhook // listener config, and optional once flag -func New(repos []*repository.Repository, hookSvr *config.HookSvrConfig, once bool) *Watcher { - repoChangeCh := make(chan *repository.Repository, len(repos)) +func New(repos []repository.Repo, hookSvr *config.HookSvrConfig, once bool) *Watcher { + repoChangeCh := make(chan repository.Repo, len(repos)) logger := log.WithField("caller", "watcher") return &Watcher{ diff --git a/watcher/webhook.go b/watcher/webhook.go index 701e3d3..ac2de7a 100644 --- a/watcher/webhook.go +++ b/watcher/webhook.go @@ -1,3 +1,19 @@ +/* +Copyright 2019 Kohl's Department Stores, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package watch import ( @@ -9,7 +25,7 @@ import ( "sync" "github.com/gorilla/mux" - "gopkg.in/libgit2/git2go.v24" + "gopkg.in/src-d/go-git.v4" ) // GithubPayload is the response from GitHub @@ -20,7 +36,7 @@ type GithubPayload struct { // StashPayload is the response from Stash type StashPayload struct { RefChanges []struct { - RefId string `json:"refId"` + RefID string `json:"refId"` } `json:"refChanges"` } @@ -126,19 +142,16 @@ func (w *Watcher) githubHandler(rw http.ResponseWriter, rq *http.Request) { repo := w.Repositories[i] w.logger.WithField("repository", repo.Name()).Info("Received hook event from GitHub") - analysis, err := repo.Pull(branchName) - if err != nil { - http.Error(rw, err.Error(), http.StatusInternalServerError) - return - } - // If there is a change, send the repo RepoChangeCh + err = repo.Pull(branchName) switch { - case analysis&git.MergeAnalysisUpToDate != 0: + case err == git.NoErrAlreadyUpToDate: w.logger.Debugf("Up to date: %s/%s", repo.Name(), branchName) - case analysis&git.MergeAnalysisNormal != 0, analysis&git.MergeAnalysisFastForward != 0: + case err == nil: w.logger.Infof("Changed: %s/%s", repo.Name(), branchName) w.RepoChangeCh <- repo + case err != nil: + w.logger.Errorf("Failed: %s/%s - %s", repo.Name(), branchName, err) } } @@ -160,7 +173,7 @@ func (w *Watcher) stashHandler(rw http.ResponseWriter, rq *http.Request) { return } - ref := payload.RefChanges[0].RefId + ref := payload.RefChanges[0].RefID if len(ref) == 0 { http.Error(rw, "ref is empty", http.StatusInternalServerError) @@ -183,19 +196,15 @@ func (w *Watcher) stashHandler(rw http.ResponseWriter, rq *http.Request) { repo := w.Repositories[i] w.logger.WithField("repository", repo.Name()).Info("Received hook event from Stash") - analysis, err := repo.Pull(branchName) - if err != nil { - http.Error(rw, err.Error(), http.StatusInternalServerError) - return - } - - // If there is a change, send the repo RepoChangeCh + err = repo.Pull(branchName) switch { - case analysis&git.MergeAnalysisUpToDate != 0: + case err == git.NoErrAlreadyUpToDate: w.logger.Debugf("Up to date: %s/%s", repo.Name(), branchName) - case analysis&git.MergeAnalysisNormal != 0, analysis&git.MergeAnalysisFastForward != 0: + case err == nil: w.logger.Infof("Changed: %s/%s", repo.Name(), branchName) w.RepoChangeCh <- repo + case err != nil: + w.logger.Errorf("Failed: %s/%s - %s", repo.Name(), branchName, err) } } @@ -250,19 +259,15 @@ func (w *Watcher) bitbucketHandler(rw http.ResponseWriter, rq *http.Request) { repo := w.Repositories[i] w.logger.WithField("repository", repo.Name()).Info("Received hook event from Bitbucket") - analysis, err := repo.Pull(branchName) - if err != nil { - http.Error(rw, err.Error(), http.StatusInternalServerError) - return - } - - // If there is a change, send the repo RepoChangeCh + err = repo.Pull(branchName) switch { - case analysis&git.MergeAnalysisUpToDate != 0: + case err == git.NoErrAlreadyUpToDate: w.logger.Debugf("Up to date: %s/%s", repo.Name(), branchName) - case analysis&git.MergeAnalysisNormal != 0, analysis&git.MergeAnalysisFastForward != 0: + case err == nil: w.logger.Infof("Changed: %s/%s", repo.Name(), branchName) w.RepoChangeCh <- repo + case err != nil: + w.logger.Errorf("Failed: %s/%s - %s", repo.Name(), branchName, err) } } @@ -316,18 +321,14 @@ func (w *Watcher) gitlabHandler(rw http.ResponseWriter, rq *http.Request) { repo := w.Repositories[i] w.logger.WithField("repository", repo.Name()).Info("Received hook event from GitLab") - analysis, err := repo.Pull(branchName) - if err != nil { - http.Error(rw, err.Error(), http.StatusInternalServerError) - return - } - - // If there is a change, send the repo RepoChangeCh + err = repo.Pull(branchName) switch { - case analysis&git.MergeAnalysisUpToDate != 0: + case err == git.NoErrAlreadyUpToDate: w.logger.Debugf("Up to date: %s/%s", repo.Name(), branchName) - case analysis&git.MergeAnalysisNormal != 0, analysis&git.MergeAnalysisFastForward != 0: + case err == nil: w.logger.Infof("Changed: %s/%s", repo.Name(), branchName) w.RepoChangeCh <- repo + case err != nil: + w.logger.Errorf("Failed: %s/%s - %s", repo.Name(), branchName, err) } }