From 5b040f0c852fcff1e90c2c13e995540a5d8c3fe6 Mon Sep 17 00:00:00 2001 From: Jakob Date: Fri, 13 Oct 2023 15:40:49 +0200 Subject: [PATCH] Update DEVELOPER_GUID, Add Coverage + go ling to Makefile, Run CI tests on all branches (#392) * .github: run test workflows for all branches, test against 2.9,2.10 Signed-off-by: Jakob Hahn * Makefile: use docker golangci-lint, run integration test on `.` folder, change coverage generation Signed-off-by: Jakob Hahn * Update DEVELOPER_GUIDE Signed-off-by: Jakob Hahn --------- Signed-off-by: Jakob Hahn --- .github/workflows/links.yml | 9 ++--- .github/workflows/lint.yml | 8 +---- .github/workflows/test-compatibility.yml | 4 ++- .github/workflows/test-integration.yml | 16 ++++----- .github/workflows/test-unit.yml | 8 +---- CHANGELOG.md | 6 ++-- DEVELOPER_GUIDE.md | 43 ++++++++++++++++++++---- Makefile | 25 ++++++++++---- 8 files changed, 75 insertions(+), 44 deletions(-) diff --git a/.github/workflows/links.yml b/.github/workflows/links.yml index a0dbc727c..f757583e9 100644 --- a/.github/workflows/links.yml +++ b/.github/workflows/links.yml @@ -1,11 +1,6 @@ name: Link Checker -on: - push: - branches: - - "*" - pull_request: - branches: - - "*" + +on: [push, pull_request] jobs: linkchecker: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 04ffe52db..50333bd37 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,12 +1,6 @@ name: Lint check -on: - push: - branches: - - "*" - pull_request: - branches: - - "*" +on: [push, pull_request] env: GITHUB_ACTIONS: true diff --git a/.github/workflows/test-compatibility.yml b/.github/workflows/test-compatibility.yml index 041e5dab2..2892b8d52 100644 --- a/.github/workflows/test-compatibility.yml +++ b/.github/workflows/test-compatibility.yml @@ -3,7 +3,7 @@ name: Integration for Compatibility on: [push, pull_request] jobs: - integration-test-compatibility: + integ-test-compat: runs-on: ubuntu-latest strategy: fail-fast: false @@ -22,6 +22,8 @@ jobs: - { opensearch_version: 2.6.0 } - { opensearch_version: 2.7.0 } - { opensearch_version: 2.8.0 } + - { opensearch_version: 2.9.0 } + - { opensearch_version: 2.10.0 } steps: - uses: actions/checkout@v3 with: { fetch-depth: 1 } diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml index 6a7c7f9dd..54e6df6a2 100644 --- a/.github/workflows/test-integration.yml +++ b/.github/workflows/test-integration.yml @@ -1,12 +1,6 @@ name: Integration -on: - push: - branches: - - "*" - pull_request: - branches: - - "*" +on: [push, pull_request] env: GITHUB_ACTIONS: true @@ -30,6 +24,9 @@ jobs: - name: Launch OpenSearch cluster run: | make cluster.clean cluster.build cluster.start + for attempt in `seq 25`; do sleep 5; \ + if curl -s localhost:9200; \ + then echo '=====> ready'; break; fi; if [ $attempt == 25 ]; then exit 1; fi; echo '=====> waiting...'; done - run: make test-integ race=true - uses: codecov/codecov-action@v1 with: @@ -57,4 +54,7 @@ jobs: - name: Launch OpenSearch cluster run: | make cluster.clean cluster.build cluster.start - - run: make test-integ-secure \ No newline at end of file + for attempt in `seq 25`; do sleep 5; \ + if curl -s -ku admin:admin https://localhost:9200; \ + then echo '=====> ready'; break; fi; if [ $attempt == 25 ]; then exit 1; fi; echo '=====> waiting...'; done + - run: make test-integ-secure diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index e5f4904f9..d09f8f376 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -1,12 +1,6 @@ name: Unit -on: - push: - branches: - - "*" - pull_request: - branches: - - "*" +on: [push, pull_request] env: GITHUB_ACTIONS: true diff --git a/CHANGELOG.md b/CHANGELOG.md index 4edf791ca..9443965fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,7 +43,9 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Read, close and replace the http Reponse Body ([#300](https://github.com/opensearch-project/opensearch-go/pull/300)) - Updated and adjusted golangci-lint, solve linting complains for signer ([#352](https://github.com/opensearch-project/opensearch-go/pull/352)) - Solve linting complains for opensearchtransport ([#353](https://github.com/opensearch-project/opensearch-go/pull/353)) -- Updated Developer guide to include docker build instructions ([#385]https://github.com/opensearch-project/opensearch-go/pull/385) +- Updated Developer guide to include docker build instructions ([#385](https://github.com/opensearch-project/opensearch-go/pull/385) +- Test against version 2.9.0,2.10.0, run tests in all branches, change intergration tests to wait for OpenSearch to start ([#392](https://github.com/opensearch-project/opensearch-go/pull/392)) +- Makefile: use docker golangci-lint, run integration test on `.` folder, change coverage generation ([#392](https://github.com/opensearch-project/opensearch-go/pull/392)) ### Deprecated @@ -59,4 +61,4 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ### Security -[Unreleased]: https://github.com/opensearch-project/opensearch-go/compare/v2.2.0...HEAD \ No newline at end of file +[Unreleased]: https://github.com/opensearch-project/opensearch-go/compare/v2.2.0...HEAD diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 60e0af8b4..843cfee8f 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -11,8 +11,11 @@ - [Execute integration tests from your terminal](#execute-integration-tests-from-your-terminal) - [Lint](#lint) - [Markdown lint](#markdown-lint) + - [Go lint](#go-lint) + - [Coverage](#coverage) - [Use an Editor](#use-an-editor) - [GoLand](#goland) + - [Vim](#vim) # Developer Guide @@ -26,9 +29,9 @@ Fork [opensearch-project/opensearch-go](https://github.com/opensearch-project/op ### Install Prerequisites -#### Go 1.11 +#### Go 1.15 -OpenSearch Go Client builds using [Go](https://golang.org/doc/install) 1.11 at a minimum. +OpenSearch Go Client builds using [Go](https://golang.org/doc/install) 1.15 at a minimum. #### Docker @@ -38,7 +41,7 @@ OpenSearch Go Client builds using [Go](https://golang.org/doc/install) 1.11 at a To build the project on Windows, use [WSL2](https://learn.microsoft.com/en-us/windows/wsl/install), the compatibility layer for running Linux applications. -Install ```make``` +Install `make` ``` sudo apt install make ``` @@ -64,11 +67,10 @@ In order to test opensearch-go client, you need a running OpenSearch cluster. Yo ### Composing an OpenSearch Docker Container -Ensure that Docker is installed on your local machine. You can check by running `docker --version`. Next, navigate to your local OpenSearch repository where the docker-compose.yaml file is stored. Run the following command to build the docker container. +Ensure that Docker is installed on your local machine. You can check by running `docker --version`. Next, navigate to your local opensearch-go repository. Run the following command to build and start the OpenSearch docker container. ``` -cd opensearch/.ci/opensearch -docker-compose -f docker-compose.yml up +make cluster.build cluster.start ``` This command will start the OpenSearch container using the `docker-compose.yaml` configuration file. During the build process, the necessary dependencies and files will be downloaded, which may take some time depending on your internet connection and system resources. @@ -104,6 +106,30 @@ To check the markdown files, run the following command: make lint.markdown ``` +### Go lint + +To check all go files, run the following command: + +``` +make linters +``` + +## Coverage + +To get the repository test coverage, run the following command: + +For the results to be display in your terminal: + +``` +make coverage +``` + +For the results to be display in your browser: + +``` +make coverage-html +``` + ## Use an Editor ### GoLand @@ -114,3 +140,8 @@ You can import the OpenSearch project into GoLand as follows: 2. In the subsequent dialog navigate to the ~/go/src/opensearch-go and click **Open** After you have opened your project, you need to specify the location of the Go SDK. You can either specify a local path to the SDK or download it. To set the Go SDK, navigate to **Go | GOROOT** and set accordingly. + +### Vim + +To improve your vim experience with Go, you might want to check out [fatih/vim-go](https://github.com/fatih/vim-go). +For example it correctly formats the file and validates it on save. diff --git a/Makefile b/Makefile index 137db3fd3..65483b95f 100644 --- a/Makefile +++ b/Makefile @@ -38,7 +38,7 @@ endif gotestsum --format=short-verbose --junitfile=tmp/integration-report.xml -- $(testintegargs) "./opensearchtransport" "./opensearchapi" "./opensearchutil"; \ else \ echo "go test -v" $(testintegargs) "."; \ - go test -v $(testintegargs) "./opensearchtransport" "./opensearchapi" "./opensearchutil"; \ + go test -v $(testintegargs) "." "./opensearchtransport" "./opensearchapi" "./opensearchutil"; \ fi; test-integ-secure: ##Run secure integration tests @@ -48,11 +48,24 @@ test-bench: ## Run benchmarks @printf "\033[2m→ Running benchmarks...\033[0m\n" go test -run=none -bench=. -benchmem ./... -test-coverage: ## Generate test coverage report +coverage: ## Print test coverage report + @make gen-coverage + @go tool cover -func=$(PWD)/tmp/total.cov + @printf "\033[0m--------------------------------------------------------------------------------\n\033[0m" + +coverage-html: ## Open test coverage report in browser + @make gen-coverage + @go tool cover -html $(PWD)/tmp/total.cov + +gen-coverage: ## Generate test coverage report @printf "\033[2m→ Generating test coverage report...\033[0m\n" - @go tool cover -html=tmp/unit.cov -o tmp/coverage.html - @go tool cover -func=tmp/unit.cov | 'grep' -v 'opensearchapi/api\.' | sed 's/github.com\/opensearch-project\/opensearch-go\///g' - @printf "\033[0m--------------------------------------------------------------------------------\nopen tmp/coverage.html\n\n\033[0m" + @rm -rf tmp + @mkdir tmp + @mkdir tmp/unit + @mkdir tmp/integration + @go test -cover ./... -args -test.gocoverdir="$(PWD)/tmp/unit" + @go test -cover -tags='integration' ./... -args -test.gocoverdir="$(PWD)/tmp/integration" + @go tool covdata textfmt -i=$(PWD)/tmp/unit,$(PWD)/tmp/integration -o $(PWD)/tmp/total.cov ##@ Development lint: ## Run lint on the package @@ -205,7 +218,7 @@ cluster.clean: ## Remove unused Docker volumes and networks docker system prune --volumes --force linters: - ./bin/golangci-lint run ./... --timeout=5m + docker run -t --rm -v $$(pwd):/app -v ~/.cache/golangci-lint/v1.53.3:/root/.cache -w /app golangci/golangci-lint:v1.53.3 golangci-lint run --timeout=5m workflow: ## Run all github workflow commands here sequentially