From c1d26f034ace4b9321c16ae89b894a3b8305bdf6 Mon Sep 17 00:00:00 2001 From: Drew Khoury Date: Fri, 7 Jan 2022 13:45:21 -0600 Subject: [PATCH 01/45] Add verification via gsd-verification-rules repo (#5) --- .github/workflows/verify.yml | 23 +++++++++++++++++++++++ .gitignore | 6 +++++- Makefile | 5 +++++ 3 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/verify.yml diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml new file mode 100644 index 00000000..677df281 --- /dev/null +++ b/.github/workflows/verify.yml @@ -0,0 +1,23 @@ +on: + push: + branches: + - master + pull_request: + types: [opened, synchronize, reopened] +name: Verify + +jobs: + verify: + runs-on: ubuntu-latest + steps: + + - name: Check out gsd-verification-rules repo + uses: actions/checkout@v2 + with: + repository: contino/gsd-verification-rules + ref: main + path: gsd-verification-rules + ssh-key: "${{ secrets.SSH_PRIVATE_KEY }}" + + - name: Run all verification rules + run: cd gsd-verification-rules && make verify diff --git a/.gitignore b/.gitignore index a4a78619..8cca8034 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,8 @@ aws.env # ide specific directories .idea -.vscode \ No newline at end of file +.vscode +Makefile + +# gsd specific +gsd-verification-rules \ No newline at end of file diff --git a/Makefile b/Makefile index c5294026..bb2ed85d 100644 --- a/Makefile +++ b/Makefile @@ -31,6 +31,11 @@ down: test: envfile ${GO_TEST_DOCKER_COMPOSE} +.PHONY: verify +verify: + git clone git@github.com:contino/gsd-verification-rules.git || true + cd gsd-verification-rules && git pull && make verify + .PHONY: create_table create_table: envfile echo "from create_table" From 8d3efcfdcfdaf89df464063b0d7bd408bd12bf6b Mon Sep 17 00:00:00 2001 From: repo-visualizer Date: Fri, 7 Jan 2022 19:45:39 +0000 Subject: [PATCH 02/45] Repo visualizer: updated diagram --- repo-visualizer.svg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo-visualizer.svg b/repo-visualizer.svg index 7a1c7327..def7ef32 100644 --- a/repo-visualizer.svg +++ b/repo-visualizer.svg @@ -1 +1 @@ -srcsrc.github/workflows.github/workflowsmain.gomain.gomain.gomain_test.gomain_test.gomain_test.goREADME.mdREADME.mdREADME.mdLICENSELICENSELICENSEMakefileMakefileMakefiledeployment.ymldeployment.ymldeployment.ymlservice.ymlservice.ymlservice.ymlkustomization...kustomization...kustomization...docker-com...docker-com...docker-com....gitignore.gitignore.gitignoresonar-pro...sonar-pro...sonar-pro...DockerfileDockerfileDockerfilemain.ymlmain.ymlmain.ymlgoogle.ymlgoogle.ymlgoogle.ymlsonar.ymlsonar.ymlsonar.ymllint.ymllint.ymllint.ymlcreate-dia...create-dia...create-dia....gitignore.go.md.mod.properties.svg.ymleach dot sized by file size \ No newline at end of file +srcsrc.github/workflows.github/workflowsmain.gomain.gomain.gomain_test.gomain_test.gomain_test.goREADME.mdREADME.mdREADME.mdLICENSELICENSELICENSEMakefileMakefileMakefiledeployment.ymldeployment.ymldeployment.ymlservice.ymlservice.ymlservice.ymlkustomization...kustomization...kustomization....gitignore.gitignore.gitignoredocker-com...docker-com...docker-com...sonar-pro...sonar-pro...sonar-pro...DockerfileDockerfileDockerfilemain.ymlmain.ymlmain.ymlgoogle.ymlgoogle.ymlgoogle.ymlsonar.ymlsonar.ymlsonar.ymllint.ymllint.ymllint.ymlverify.ymlverify.ymlverify.ymlcreate-dia...create-dia...create-dia....gitignore.go.md.mod.properties.svg.ymleach dot sized by file size \ No newline at end of file From 79a8f21681c1ebe4b3a696d9d81d633315138ddb Mon Sep 17 00:00:00 2001 From: Drew Khoury Date: Sat, 15 Jan 2022 13:58:46 -0600 Subject: [PATCH 03/45] adding pipeline id to artifact metadata (#7) * adding pipeline id to artifact metadata * using v2 table def/name * added help to makefile --- .github/workflows/main-test.yml.backup | 39 ++++++++++++++++++ .github/workflows/main.yml | 55 +++++++------------------- Makefile | 25 ++++++------ 3 files changed, 66 insertions(+), 53 deletions(-) create mode 100644 .github/workflows/main-test.yml.backup diff --git a/.github/workflows/main-test.yml.backup b/.github/workflows/main-test.yml.backup new file mode 100644 index 00000000..b563601d --- /dev/null +++ b/.github/workflows/main-test.yml.backup @@ -0,0 +1,39 @@ +# Used to test things during PR's that would normally only happen on main branch +# +# eg updates to dynamodb table +# + +name: Main CI (PR TEST) + +on: + pull_request: + types: [opened, synchronize, reopened] + +env: + IMAGE_NAME: go-hello-world + GITHUB_TOKEN: ${{ secrets.GITHUBTOKEN }} + +jobs: + + buildtest: + runs-on: ubuntu-latest + + steps: + + - name: checkout repo + uses: actions/checkout@v2 + + - name: Create DynamboDB Table in AWS - To store metadata (one-time) + run: make create_table + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: us-east-2 + + # - name: Create tags in DynamboDB Table in AWS - Metadata for this commit + # run: make create_tags + # env: + # PIPELINE_ID: ${GITHUB_RUN_ID}-${GITHUB_RUN_NUMBER} + # AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # AWS_DEFAULT_REGION: us-east-2 \ No newline at end of file diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ef9c53b8..6fa00a2c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,4 +1,4 @@ -name: CI +name: Main CI on: push: @@ -46,25 +46,8 @@ jobs: if: github.event_name == 'push' steps: - - uses: actions/checkout@v2 - - # this will cause a failure which is only in some demos but annoying in others - #- uses: ynniss/golang-security-action@master - # with: - # #CODE_PATH: "./src/" # <<< we should move our go source into a generic src container so the refernce app is nicely organized and it's easier to build/find source - # CODE_PATH: "./" - - - - - - - - - - - - + - name: checkout code + uses: actions/checkout@v2 - name: Tests run: make test @@ -75,32 +58,24 @@ jobs: - name: Run the Go package locally (detached) run: make run - #- name: Create DynamboDB Table in AWS - # run: make create_table - # env: - # FOO: ${{ secrets.FOO }} - # BAR: "BAZ" - # AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # AWS_DEFAULT_REGION: us-east-2 - - - name: Create tags in DynamboDB Table in AWS + # only used to initially create the table - need a cleaner way to include this, + # and have it do nothing if the table exists + # + # - name: Create DynamboDB Table in AWS - To store metadata (one-time) + # run: make create_table + # env: + # AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # AWS_DEFAULT_REGION: us-east-2 + + - name: Create tags in DynamboDB Table in AWS - Metadata for this commit run: make create_tags env: - FOO: ${{ secrets.FOO }} - BAR: "BAZ" + PIPELINE_ID: ${GITHUB_RUN_ID} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_DEFAULT_REGION: us-east-2 - - - - - - - - # this was just in here to prove our the publish, should actually edit the make build do write the correct image name then we can remove this step - name: Build image run: docker build . --file Dockerfile --tag $IMAGE_NAME diff --git a/Makefile b/Makefile index bb2ed85d..31155d2d 100644 --- a/Makefile +++ b/Makefile @@ -1,14 +1,19 @@ DOCKER_TAG ?= go-hello-world FULL_TAG ?= ${DOCKER_TAG}:${HASH} -DYNAMODB_TABLE ?= ${DOCKER_TAG} +DYNAMODB_TABLE ?= ${DOCKER_TAG}-v2 PORT ?= "8080" GO_TEST_DOCKER_COMPOSE ?= docker-compose run --rm gobase go test -v -cover AWS_CLI_DOCKER_COMPOSE ?= docker-compose run --rm awscli HASH := $(shell git rev-parse HEAD) -VERACODE_ID?= "someveracodeid" +PIPELINE_BASE?= contino/gsd-hello-world ENVFILE ?= aws.template +.DEFAULT_GOAL := help + +help: + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' + envfile: echo "from envfile" echo "FOO=${FOO}" @@ -42,21 +47,15 @@ create_table: envfile echo "FOO=${FOO}" echo "BAR=${BAR}" ${AWS_CLI_DOCKER_COMPOSE} dynamodb create-table \ - --table-name ${DYNAMODB_TABLE} \ - --attribute-definitions \ - AttributeName=GIT_COMMIT,AttributeType=S \ - AttributeName=VERACODE_ID,AttributeType=S \ - --key-schema \ - AttributeName=GIT_COMMIT,KeyType=HASH \ - AttributeName=VERACODE_ID,KeyType=RANGE \ - --provisioned-throughput \ - ReadCapacityUnits=10,WriteCapacityUnits=5 + --table-name ${DYNAMODB_TABLE} \ + --attribute-definitions AttributeName=GIT_COMMIT,AttributeType=S AttributeName=PIPELINE_ID,AttributeType=S \ + --key-schema AttributeName=GIT_COMMIT,KeyType=HASH AttributeName=PIPELINE_ID,KeyType=RANGE \ + --provisioned-throughput ReadCapacityUnits=10,WriteCapacityUnits=5 create_tags: envfile ${AWS_CLI_DOCKER_COMPOSE} dynamodb put-item \ --table-name ${DYNAMODB_TABLE} \ - --item \ - '{"GIT_COMMIT": {"S": "${HASH}"}, "VERACODE_ID":{"S": ${VERACODE_ID}}}' + --item '{ "GIT_COMMIT": {"S": "${HASH}"}, "PIPELINE_BASE":{"S": "${PIPELINE_BASE}"}, "PIPELINE_ID":{"S": "${PIPELINE_ID}"} }' .PHONY: clean clean: From 2b81dd8122f4850851d6a196a7591223cd22ca13 Mon Sep 17 00:00:00 2001 From: repo-visualizer Date: Sat, 15 Jan 2022 19:59:01 +0000 Subject: [PATCH 04/45] Repo visualizer: updated diagram --- repo-visualizer.svg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo-visualizer.svg b/repo-visualizer.svg index def7ef32..ad1809dc 100644 --- a/repo-visualizer.svg +++ b/repo-visualizer.svg @@ -1 +1 @@ -srcsrc.github/workflows.github/workflowsmain.gomain.gomain.gomain_test.gomain_test.gomain_test.goREADME.mdREADME.mdREADME.mdLICENSELICENSELICENSEMakefileMakefileMakefiledeployment.ymldeployment.ymldeployment.ymlservice.ymlservice.ymlservice.ymlkustomization...kustomization...kustomization....gitignore.gitignore.gitignoredocker-com...docker-com...docker-com...sonar-pro...sonar-pro...sonar-pro...DockerfileDockerfileDockerfilemain.ymlmain.ymlmain.ymlgoogle.ymlgoogle.ymlgoogle.ymlsonar.ymlsonar.ymlsonar.ymllint.ymllint.ymllint.ymlverify.ymlverify.ymlverify.ymlcreate-dia...create-dia...create-dia....gitignore.go.md.mod.properties.svg.ymleach dot sized by file size \ No newline at end of file +srcsrc.github/workflows.github/workflowsmain.gomain.gomain.gomain_test.gomain_test.gomain_test.goREADME.mdREADME.mdREADME.mdLICENSELICENSELICENSEMakefileMakefileMakefiledeployment.ymldeployment.ymldeployment.ymlservice.ymlservice.ymlservice.ymlkustomizatio...kustomizatio...kustomizatio....gitignore.gitignore.gitignoredocker-com...docker-com...docker-com...sonar-pro...sonar-pro...sonar-pro...DockerfileDockerfileDockerfilegoogle.ymlgoogle.ymlgoogle.ymlmain.ymlmain.ymlmain.ymlmain-test.yml.b...main-test.yml.b...main-test.yml.b...sonar.ymlsonar.ymlsonar.ymllint.ymllint.ymllint.ymlverify.ymlverify.ymlverify.ymlcreate-dia...create-dia...create-dia....gitignore.go.md.mod.properties.svg.ymleach dot sized by file size \ No newline at end of file From f06f3d88bc8dfaef3057b7210278e1f6e2a4635a Mon Sep 17 00:00:00 2001 From: Drew Khoury Date: Sat, 15 Jan 2022 18:34:33 -0600 Subject: [PATCH 05/45] master main rename (#10) --- .github/workflows/create-diagram.yml | 2 +- .github/workflows/google.yml | 2 +- .github/workflows/lint.yml | 2 +- .github/workflows/main-test.yml.backup | 2 +- .github/workflows/main.yml | 2 +- .github/workflows/sonar.yml | 2 +- .github/workflows/verify.yml | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/create-diagram.yml b/.github/workflows/create-diagram.yml index f45245e5..8d4fffc2 100644 --- a/.github/workflows/create-diagram.yml +++ b/.github/workflows/create-diagram.yml @@ -4,7 +4,7 @@ on: tags: - v* branches: - - master + - main pull_request: jobs: repo-visuals: diff --git a/.github/workflows/google.yml b/.github/workflows/google.yml index 28e42554..15761151 100644 --- a/.github/workflows/google.yml +++ b/.github/workflows/google.yml @@ -18,7 +18,7 @@ name: Build and Deploy to GKE on: push: branches: - - master + - main env: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index f7ce46c6..83fb3e46 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -4,7 +4,7 @@ on: tags: - v* branches: - - master + - main pull_request: jobs: lint: diff --git a/.github/workflows/main-test.yml.backup b/.github/workflows/main-test.yml.backup index b563601d..4eb753df 100644 --- a/.github/workflows/main-test.yml.backup +++ b/.github/workflows/main-test.yml.backup @@ -3,7 +3,7 @@ # eg updates to dynamodb table # -name: Main CI (PR TEST) +name: Main CI (TESTING for PRs) on: pull_request: diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6fa00a2c..6403011f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -4,7 +4,7 @@ on: push: # Publish `master` as Docker `latest` image. branches: - - master + - main # Publish `v1.2.3` tags as releases. tags: diff --git a/.github/workflows/sonar.yml b/.github/workflows/sonar.yml index b6e23b6f..caa60199 100644 --- a/.github/workflows/sonar.yml +++ b/.github/workflows/sonar.yml @@ -1,7 +1,7 @@ on: push: branches: - - master + - main pull_request: types: [opened, synchronize, reopened] name: Code Quality diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 677df281..10c93738 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -1,7 +1,7 @@ on: push: branches: - - master + - main pull_request: types: [opened, synchronize, reopened] name: Verify From 6d336fdec3b649da2ba1bf9fe20f5c41a11b0b8b Mon Sep 17 00:00:00 2001 From: repo-visualizer Date: Sun, 16 Jan 2022 00:34:49 +0000 Subject: [PATCH 06/45] Repo visualizer: updated diagram --- repo-visualizer.svg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo-visualizer.svg b/repo-visualizer.svg index ad1809dc..ba72ce3a 100644 --- a/repo-visualizer.svg +++ b/repo-visualizer.svg @@ -1 +1 @@ -srcsrc.github/workflows.github/workflowsmain.gomain.gomain.gomain_test.gomain_test.gomain_test.goREADME.mdREADME.mdREADME.mdLICENSELICENSELICENSEMakefileMakefileMakefiledeployment.ymldeployment.ymldeployment.ymlservice.ymlservice.ymlservice.ymlkustomizatio...kustomizatio...kustomizatio....gitignore.gitignore.gitignoredocker-com...docker-com...docker-com...sonar-pro...sonar-pro...sonar-pro...DockerfileDockerfileDockerfilegoogle.ymlgoogle.ymlgoogle.ymlmain.ymlmain.ymlmain.ymlmain-test.yml.b...main-test.yml.b...main-test.yml.b...sonar.ymlsonar.ymlsonar.ymllint.ymllint.ymllint.ymlverify.ymlverify.ymlverify.ymlcreate-dia...create-dia...create-dia....gitignore.go.md.mod.properties.svg.ymleach dot sized by file size \ No newline at end of file +srcsrc.github/workflows.github/workflowsmain.gomain.gomain.gomain_test.gomain_test.gomain_test.goREADME.mdREADME.mdREADME.mdLICENSELICENSELICENSEMakefileMakefileMakefiledeployment.ymldeployment.ymldeployment.ymlservice.ymlservice.ymlservice.ymlkustomizatio...kustomizatio...kustomizatio....gitignore.gitignore.gitignoredocker-com...docker-com...docker-com...sonar-pro...sonar-pro...sonar-pro...DockerfileDockerfileDockerfilegoogle.ymlgoogle.ymlgoogle.ymlmain.ymlmain.ymlmain.ymlmain-test.yml.b...main-test.yml.b...main-test.yml.b...sonar.ymlsonar.ymlsonar.ymllint.ymllint.ymllint.ymlverify.ymlverify.ymlverify.ymlcreate-dia...create-dia...create-dia....gitignore.go.md.mod.properties.svg.ymleach dot sized by file size \ No newline at end of file From 39aa809d4324d47fee4dd8cf75b42bf30595beed Mon Sep 17 00:00:00 2001 From: Drew Khoury Date: Sat, 15 Jan 2022 18:35:29 -0600 Subject: [PATCH 07/45] adding sonar properties to point to demo project (#9) * sonar properties * Update sonar-project.properties --- .github/workflows/sonar.yml | 11 +++-------- sonar-project.properties | 2 +- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/.github/workflows/sonar.yml b/.github/workflows/sonar.yml index caa60199..d4d0ac2d 100644 --- a/.github/workflows/sonar.yml +++ b/.github/workflows/sonar.yml @@ -18,14 +18,9 @@ jobs: uses: sonarsource/sonarcloud-github-action@master with: projectBaseDir: src -# args: > -# -Dsonar.organization=my-organization -# -Dsonar.projectKey=my-projectkey -# -Dsonar.python.coverage.reportPaths=coverage.xml -# -Dsonar.sources=lib/ -# -Dsonar.test.exclusions=tests/** -# -Dsonar.tests=tests/ -# -Dsonar.verbose=true + args: > + -Dsonar.organization=fooooooo + -Dsonar.projectKey=barrrrrrrrr env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} diff --git a/sonar-project.properties b/sonar-project.properties index a259be2c..6959fee7 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -4,4 +4,4 @@ sonar.projectKey=barrrrrrrrr # relative paths to source directories. More details and properties are described # in https://sonarcloud.io/documentation/project-administration/narrowing-the-focus/ -sonar.sources=. +sonar.sources=./src From 728972cc87826dca96482191d43fe635d730f1e6 Mon Sep 17 00:00:00 2001 From: repo-visualizer Date: Sun, 16 Jan 2022 00:35:49 +0000 Subject: [PATCH 08/45] Repo visualizer: updated diagram --- repo-visualizer.svg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo-visualizer.svg b/repo-visualizer.svg index ba72ce3a..c1df9699 100644 --- a/repo-visualizer.svg +++ b/repo-visualizer.svg @@ -1 +1 @@ -srcsrc.github/workflows.github/workflowsmain.gomain.gomain.gomain_test.gomain_test.gomain_test.goREADME.mdREADME.mdREADME.mdLICENSELICENSELICENSEMakefileMakefileMakefiledeployment.ymldeployment.ymldeployment.ymlservice.ymlservice.ymlservice.ymlkustomizatio...kustomizatio...kustomizatio....gitignore.gitignore.gitignoredocker-com...docker-com...docker-com...sonar-pro...sonar-pro...sonar-pro...DockerfileDockerfileDockerfilegoogle.ymlgoogle.ymlgoogle.ymlmain.ymlmain.ymlmain.ymlmain-test.yml.b...main-test.yml.b...main-test.yml.b...sonar.ymlsonar.ymlsonar.ymllint.ymllint.ymllint.ymlverify.ymlverify.ymlverify.ymlcreate-dia...create-dia...create-dia....gitignore.go.md.mod.properties.svg.ymleach dot sized by file size \ No newline at end of file +srcsrc.github/workflows.github/workflowsmain.gomain.gomain.gomain_test.gomain_test.gomain_test.goREADME.mdREADME.mdREADME.mdLICENSELICENSELICENSEMakefileMakefileMakefiledeployment.ymldeployment.ymldeployment.ymlservice.ymlservice.ymlservice.ymlkustomization...kustomization...kustomization....gitignore.gitignore.gitignoredocker-com...docker-com...docker-com...sonar-pro...sonar-pro...sonar-pro...DockerfileDockerfileDockerfilegoogle.ymlgoogle.ymlgoogle.ymlmain.ymlmain.ymlmain.ymlmain-test.yml.b...main-test.yml.b...main-test.yml.b...lint.ymllint.ymllint.ymlsonar.ymlsonar.ymlsonar.ymlverify.ymlverify.ymlverify.ymlcreate-dia...create-dia...create-dia....gitignore.go.md.mod.properties.svg.ymleach dot sized by file size \ No newline at end of file From 0314890d92d4a2b27b58f46b3e013204eae1b8b6 Mon Sep 17 00:00:00 2001 From: Drew Khoury Date: Sat, 15 Jan 2022 18:36:54 -0600 Subject: [PATCH 09/45] update test threshold and coverage message (#8) --- src/main_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main_test.go b/src/main_test.go index 8e6531dd..6c70cd32 100644 --- a/src/main_test.go +++ b/src/main_test.go @@ -43,8 +43,8 @@ func TestGETHome(t *testing.T) { if testing.CoverMode() != "" { c := testing.Coverage() - if c < 0.8 { - fmt.Println("Tests passed but coverage failed at", c) + if c < 0.15 { + fmt.Println("Tests passed but test-coverage below threshold of at least 15%. Current test-coverage is: ", c) rc = -1 } } From d0883c8178833374852af3301bffbb5e6dd5fd82 Mon Sep 17 00:00:00 2001 From: repo-visualizer Date: Sun, 16 Jan 2022 00:37:10 +0000 Subject: [PATCH 10/45] Repo visualizer: updated diagram --- repo-visualizer.svg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo-visualizer.svg b/repo-visualizer.svg index c1df9699..25c5d989 100644 --- a/repo-visualizer.svg +++ b/repo-visualizer.svg @@ -1 +1 @@ -srcsrc.github/workflows.github/workflowsmain.gomain.gomain.gomain_test.gomain_test.gomain_test.goREADME.mdREADME.mdREADME.mdLICENSELICENSELICENSEMakefileMakefileMakefiledeployment.ymldeployment.ymldeployment.ymlservice.ymlservice.ymlservice.ymlkustomization...kustomization...kustomization....gitignore.gitignore.gitignoredocker-com...docker-com...docker-com...sonar-pro...sonar-pro...sonar-pro...DockerfileDockerfileDockerfilegoogle.ymlgoogle.ymlgoogle.ymlmain.ymlmain.ymlmain.ymlmain-test.yml.b...main-test.yml.b...main-test.yml.b...lint.ymllint.ymllint.ymlsonar.ymlsonar.ymlsonar.ymlverify.ymlverify.ymlverify.ymlcreate-dia...create-dia...create-dia....gitignore.go.md.mod.properties.svg.ymleach dot sized by file size \ No newline at end of file +srcsrc.github/workflows.github/workflowsmain_test.gomain_test.gomain_test.gomain.gomain.gomain.goREADME.mdREADME.mdREADME.mdLICENSELICENSELICENSEMakefileMakefileMakefiledeployment.ymldeployment.ymldeployment.ymlservice.ymlservice.ymlservice.ymlkustomization...kustomization...kustomization....gitignore.gitignore.gitignoredocker-com...docker-com...docker-com...sonar-pro...sonar-pro...sonar-pro...DockerfileDockerfileDockerfilegoogle.ymlgoogle.ymlgoogle.ymlmain.ymlmain.ymlmain.ymlmain-test.yml.b...main-test.yml.b...main-test.yml.b...lint.ymllint.ymllint.ymlsonar.ymlsonar.ymlsonar.ymlverify.ymlverify.ymlverify.ymlcreate-dia...create-dia...create-dia....gitignore.go.md.mod.properties.svg.ymleach dot sized by file size \ No newline at end of file From 3b102c84d3ba7a726dec7488db35404b90336eba Mon Sep 17 00:00:00 2001 From: Drew Khoury Date: Sat, 15 Jan 2022 19:25:16 -0600 Subject: [PATCH 11/45] fix logic so test suit and set pass rate to be able to show how it passes (#12) Co-authored-by: drew --- src/main_test.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main_test.go b/src/main_test.go index 6c70cd32..488173f9 100644 --- a/src/main_test.go +++ b/src/main_test.go @@ -43,12 +43,14 @@ func TestGETHome(t *testing.T) { if testing.CoverMode() != "" { c := testing.Coverage() - if c < 0.15 { - fmt.Println("Tests passed but test-coverage below threshold of at least 15%. Current test-coverage is: ", c) + if c < 0.1 { + fmt.Println("Tests passed but test-coverage below threshold of less than 10%. Current test-coverage is: ", c) rc = -1 + os.Exit(rc) } + if c >= 0.1 { + fmt.Println("Tests passed and test-coverage is above threshold of at least 10%. Current test-coverage is: ", c) + } } - os.Exit(rc) - } From cf5b75b9e143653c86ae2616edd82c8a266da0a2 Mon Sep 17 00:00:00 2001 From: repo-visualizer Date: Sun, 16 Jan 2022 01:25:32 +0000 Subject: [PATCH 12/45] Repo visualizer: updated diagram --- repo-visualizer.svg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo-visualizer.svg b/repo-visualizer.svg index 25c5d989..4407a220 100644 --- a/repo-visualizer.svg +++ b/repo-visualizer.svg @@ -1 +1 @@ -srcsrc.github/workflows.github/workflowsmain_test.gomain_test.gomain_test.gomain.gomain.gomain.goREADME.mdREADME.mdREADME.mdLICENSELICENSELICENSEMakefileMakefileMakefiledeployment.ymldeployment.ymldeployment.ymlservice.ymlservice.ymlservice.ymlkustomization...kustomization...kustomization....gitignore.gitignore.gitignoredocker-com...docker-com...docker-com...sonar-pro...sonar-pro...sonar-pro...DockerfileDockerfileDockerfilegoogle.ymlgoogle.ymlgoogle.ymlmain.ymlmain.ymlmain.ymlmain-test.yml.b...main-test.yml.b...main-test.yml.b...lint.ymllint.ymllint.ymlsonar.ymlsonar.ymlsonar.ymlverify.ymlverify.ymlverify.ymlcreate-dia...create-dia...create-dia....gitignore.go.md.mod.properties.svg.ymleach dot sized by file size \ No newline at end of file +srcsrc.github/workflows.github/workflowsmain_test.gomain_test.gomain_test.gomain.gomain.gomain.goREADME.mdREADME.mdREADME.mdLICENSELICENSELICENSEMakefileMakefileMakefiledeployment.ymldeployment.ymldeployment.ymlservice.ymlservice.ymlservice.ymlkustomization...kustomization...kustomization....gitignore.gitignore.gitignoredocker-com...docker-com...docker-com...sonar-pro...sonar-pro...sonar-pro...DockerfileDockerfileDockerfilegoogle.ymlgoogle.ymlgoogle.ymlmain.ymlmain.ymlmain.ymlmain-test.yml.b...main-test.yml.b...main-test.yml.b...lint.ymllint.ymllint.ymlsonar.ymlsonar.ymlsonar.ymlverify.ymlverify.ymlverify.ymlcreate-dia...create-dia...create-dia....gitignore.go.md.mod.properties.svg.ymleach dot sized by file size \ No newline at end of file From 111d0de449380feae500959f6b771c16a66e667f Mon Sep 17 00:00:00 2001 From: Drew Khoury Date: Sat, 15 Jan 2022 19:43:42 -0600 Subject: [PATCH 13/45] Update main.yml (#13) --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6403011f..5e47f655 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -83,7 +83,7 @@ jobs: # this is a bit verbose so probably time we wrapped it in make (it was just a copy/paste from github actions anyway) - name: Log into registry and Push image run: | - echo "${{ secrets.GH_PACKAGES }}" | docker login docker.pkg.github.com -u ${{ github.actor }} --password-stdin + echo "${{ secrets.GH_PACKAGES_2 }}" | docker login docker.pkg.github.com -u ${{ github.actor }} --password-stdin IMAGE_ID=docker.pkg.github.com/${{ github.repository }}/$IMAGE_NAME From 7956c50a5c30826c1361245b41a6ad76eba5f0f9 Mon Sep 17 00:00:00 2001 From: repo-visualizer Date: Sun, 16 Jan 2022 01:43:56 +0000 Subject: [PATCH 14/45] Repo visualizer: updated diagram --- repo-visualizer.svg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo-visualizer.svg b/repo-visualizer.svg index 4407a220..a06d3e32 100644 --- a/repo-visualizer.svg +++ b/repo-visualizer.svg @@ -1 +1 @@ -srcsrc.github/workflows.github/workflowsmain_test.gomain_test.gomain_test.gomain.gomain.gomain.goREADME.mdREADME.mdREADME.mdLICENSELICENSELICENSEMakefileMakefileMakefiledeployment.ymldeployment.ymldeployment.ymlservice.ymlservice.ymlservice.ymlkustomization...kustomization...kustomization....gitignore.gitignore.gitignoredocker-com...docker-com...docker-com...sonar-pro...sonar-pro...sonar-pro...DockerfileDockerfileDockerfilegoogle.ymlgoogle.ymlgoogle.ymlmain.ymlmain.ymlmain.ymlmain-test.yml.b...main-test.yml.b...main-test.yml.b...lint.ymllint.ymllint.ymlsonar.ymlsonar.ymlsonar.ymlverify.ymlverify.ymlverify.ymlcreate-dia...create-dia...create-dia....gitignore.go.md.mod.properties.svg.ymleach dot sized by file size \ No newline at end of file +srcsrc.github/workflows.github/workflowsmain_test.gomain_test.gomain_test.gomain.gomain.gomain.goREADME.mdREADME.mdREADME.mdLICENSELICENSELICENSEMakefileMakefileMakefiledeployment.ymldeployment.ymldeployment.ymlservice.ymlservice.ymlservice.ymlkustomization...kustomization...kustomization....gitignore.gitignore.gitignoredocker-com...docker-com...docker-com...sonar-pro...sonar-pro...sonar-pro...DockerfileDockerfileDockerfilegoogle.ymlgoogle.ymlgoogle.ymlmain.ymlmain.ymlmain.ymlmain-test.yml.b...main-test.yml.b...main-test.yml.b...lint.ymllint.ymllint.ymlsonar.ymlsonar.ymlsonar.ymlverify.ymlverify.ymlverify.ymlcreate-dia...create-dia...create-dia....gitignore.go.md.mod.properties.svg.ymleach dot sized by file size \ No newline at end of file From a6270009aa8a4153bf77f1a693fadc91e6965e2f Mon Sep 17 00:00:00 2001 From: Drew Khoury Date: Sat, 15 Jan 2022 20:03:29 -0600 Subject: [PATCH 15/45] remove sec steps from main (#14) * Update main.yml * Create sec.yml * Update and rename sonar.yml to quality.yml * Delete lint.yml * Rename google.yml to deploy.yml * run each time --- .github/workflows/{google.yml => deploy.yml} | 0 .github/workflows/main.yml | 21 ----------- .github/workflows/{lint.yml => quality.yml} | 26 ++++++++++++-- .github/workflows/sec.yml | 37 ++++++++++++++++++++ .github/workflows/sonar.yml | 26 -------------- 5 files changed, 60 insertions(+), 50 deletions(-) rename .github/workflows/{google.yml => deploy.yml} (100%) rename .github/workflows/{lint.yml => quality.yml} (54%) create mode 100644 .github/workflows/sec.yml delete mode 100644 .github/workflows/sonar.yml diff --git a/.github/workflows/google.yml b/.github/workflows/deploy.yml similarity index 100% rename from .github/workflows/google.yml rename to .github/workflows/deploy.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5e47f655..a49a1ab4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -18,27 +18,6 @@ env: GITHUB_TOKEN: ${{ secrets.GITHUBTOKEN }} # not sure why gh didn't let me create a secret with an underscore in it???????? jobs: - security: - runs-on: ubuntu-latest - if: github.event_name == 'push' - - steps: - - uses: actions/checkout@v2 - - uses: ynniss/golang-security-action@master - with: - CODE_PATH: "./src/" - - security-gosec: - runs-on: ubuntu-latest - env: - GO111MODULE: on - steps: - - name: Checkout Source - uses: actions/checkout@v2 - - name: Run Gosec Security Scanner - uses: securego/gosec@master - with: - args: ./... build: diff --git a/.github/workflows/lint.yml b/.github/workflows/quality.yml similarity index 54% rename from .github/workflows/lint.yml rename to .github/workflows/quality.yml index 83fb3e46..744521f0 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/quality.yml @@ -1,12 +1,14 @@ -name: lint on: push: - tags: - - v* branches: - main pull_request: + types: [opened, synchronize, reopened] + +name: Code Quality + jobs: + lint: name: lint runs-on: ubuntu-latest @@ -26,3 +28,21 @@ jobs: # Optional: show only new issues if it's a pull request. The default value is `false`. # only-new-issues: true + + sonarcloud: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + # Disabling shallow clone is recommended for improving relevancy of reporting + fetch-depth: 0 + - name: SonarCloud Scan + uses: sonarsource/sonarcloud-github-action@master + with: + projectBaseDir: src + args: > + -Dsonar.organization=fooooooo + -Dsonar.projectKey=barrrrrrrrr + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} diff --git a/.github/workflows/sec.yml b/.github/workflows/sec.yml new file mode 100644 index 00000000..f1683efd --- /dev/null +++ b/.github/workflows/sec.yml @@ -0,0 +1,37 @@ +name: Security + +on: + push: + # Publish `master` as Docker `latest` image. + branches: + - master + + # Publish `v1.2.3` tags as releases. + tags: + - v* + + # Run tests for any PRs. + pull_request: + +jobs: + + golang-security: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - uses: ynniss/golang-security-action@master + with: + CODE_PATH: "./src/" + + gosec-scanner: + runs-on: ubuntu-latest + env: + GO111MODULE: on + steps: + - name: Checkout Source + uses: actions/checkout@v2 + - name: Run Gosec Security Scanner + uses: securego/gosec@master + with: + args: ./... diff --git a/.github/workflows/sonar.yml b/.github/workflows/sonar.yml deleted file mode 100644 index d4d0ac2d..00000000 --- a/.github/workflows/sonar.yml +++ /dev/null @@ -1,26 +0,0 @@ -on: - push: - branches: - - main - pull_request: - types: [opened, synchronize, reopened] -name: Code Quality - -jobs: - sonarcloud: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - # Disabling shallow clone is recommended for improving relevancy of reporting - fetch-depth: 0 - - name: SonarCloud Scan - uses: sonarsource/sonarcloud-github-action@master - with: - projectBaseDir: src - args: > - -Dsonar.organization=fooooooo - -Dsonar.projectKey=barrrrrrrrr - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} From 017eb9c332378c839ad2ff4a2d3bdf05d4a814f1 Mon Sep 17 00:00:00 2001 From: repo-visualizer Date: Sun, 16 Jan 2022 02:03:42 +0000 Subject: [PATCH 16/45] Repo visualizer: updated diagram --- repo-visualizer.svg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo-visualizer.svg b/repo-visualizer.svg index a06d3e32..40fc94ac 100644 --- a/repo-visualizer.svg +++ b/repo-visualizer.svg @@ -1 +1 @@ -srcsrc.github/workflows.github/workflowsmain_test.gomain_test.gomain_test.gomain.gomain.gomain.goREADME.mdREADME.mdREADME.mdLICENSELICENSELICENSEMakefileMakefileMakefiledeployment.ymldeployment.ymldeployment.ymlservice.ymlservice.ymlservice.ymlkustomization...kustomization...kustomization....gitignore.gitignore.gitignoredocker-com...docker-com...docker-com...sonar-pro...sonar-pro...sonar-pro...DockerfileDockerfileDockerfilegoogle.ymlgoogle.ymlgoogle.ymlmain.ymlmain.ymlmain.ymlmain-test.yml.b...main-test.yml.b...main-test.yml.b...lint.ymllint.ymllint.ymlsonar.ymlsonar.ymlsonar.ymlverify.ymlverify.ymlverify.ymlcreate-dia...create-dia...create-dia....gitignore.go.md.mod.properties.svg.ymleach dot sized by file size \ No newline at end of file +srcsrc.github/workflows.github/workflowsmain_test.gomain_test.gomain_test.gomain.gomain.gomain.goREADME.mdREADME.mdREADME.mdLICENSELICENSELICENSEMakefileMakefileMakefiledeployment.ymldeployment.ymldeployment.ymlservice.ymlservice.ymlservice.ymlkustomization...kustomization...kustomization....gitignore.gitignore.gitignoredocker-com...docker-com...docker-com...sonar-pro...sonar-pro...sonar-pro...DockerfileDockerfileDockerfiledeploy.ymldeploy.ymldeploy.ymlmain.ymlmain.ymlmain.ymlquality.ymlquality.ymlquality.ymlmain-test.yml.b...main-test.yml.b...main-test.yml.b...sec.ymlsec.ymlsec.ymlverify.ymlverify.ymlverify.ymlcreate-dia...create-dia...create-dia....gitignore.go.md.mod.properties.svg.ymleach dot sized by file size \ No newline at end of file From 11fbd7e8af954762eef38f27ec1d101d31df73dd Mon Sep 17 00:00:00 2001 From: James Noonan Date: Tue, 18 Jan 2022 13:16:05 -0500 Subject: [PATCH 17/45] add new test to test the /events endpoint (#15) --- src/main_test.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/main_test.go b/src/main_test.go index 488173f9..171d8871 100644 --- a/src/main_test.go +++ b/src/main_test.go @@ -54,3 +54,21 @@ func TestGETHome(t *testing.T) { } } + +func TestGetOneEvent(t *testing.T) { + t.Run("returns 200 status code", func(t *testing.T) { + request, _ := http.NewRequest(http.MethodGet, "/events/1", nil) + response := httptest.NewRecorder() + + getOneEvent(response, request) + + got := response.Result().StatusCode + want := 200 + + if got != want { + t.Errorf("got %q, want %q", got, want) + } + }) + + +} \ No newline at end of file From 56b55c783c3b603ee5d5cce177229b66985abce0 Mon Sep 17 00:00:00 2001 From: repo-visualizer Date: Tue, 18 Jan 2022 18:16:23 +0000 Subject: [PATCH 18/45] Repo visualizer: updated diagram --- repo-visualizer.svg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo-visualizer.svg b/repo-visualizer.svg index 40fc94ac..faf427f7 100644 --- a/repo-visualizer.svg +++ b/repo-visualizer.svg @@ -1 +1 @@ -srcsrc.github/workflows.github/workflowsmain_test.gomain_test.gomain_test.gomain.gomain.gomain.goREADME.mdREADME.mdREADME.mdLICENSELICENSELICENSEMakefileMakefileMakefiledeployment.ymldeployment.ymldeployment.ymlservice.ymlservice.ymlservice.ymlkustomization...kustomization...kustomization....gitignore.gitignore.gitignoredocker-com...docker-com...docker-com...sonar-pro...sonar-pro...sonar-pro...DockerfileDockerfileDockerfiledeploy.ymldeploy.ymldeploy.ymlmain.ymlmain.ymlmain.ymlquality.ymlquality.ymlquality.ymlmain-test.yml.b...main-test.yml.b...main-test.yml.b...sec.ymlsec.ymlsec.ymlverify.ymlverify.ymlverify.ymlcreate-dia...create-dia...create-dia....gitignore.go.md.mod.properties.svg.ymleach dot sized by file size \ No newline at end of file +srcsrc.github/workflows.github/workflowsmain_test.gomain_test.gomain_test.gomain.gomain.gomain.goREADME.mdREADME.mdREADME.mdLICENSELICENSELICENSEMakefileMakefileMakefiledeployment.ymldeployment.ymldeployment.ymlservice.ymlservice.ymlservice.ymlkustomization...kustomization...kustomization....gitignore.gitignore.gitignoredocker-com...docker-com...docker-com...sonar-pro...sonar-pro...sonar-pro...DockerfileDockerfileDockerfiledeploy.ymldeploy.ymldeploy.ymlmain.ymlmain.ymlmain.ymlquality.ymlquality.ymlquality.ymlmain-test.yml.b...main-test.yml.b...main-test.yml.b...sec.ymlsec.ymlsec.ymlverify.ymlverify.ymlverify.ymlcreate-dia...create-dia...create-dia....gitignore.go.md.mod.properties.svg.ymleach dot sized by file size \ No newline at end of file From 040b9b221fad43f16d5a1ffec666a880d8d20e6e Mon Sep 17 00:00:00 2001 From: Drew Khoury Date: Tue, 5 Apr 2022 16:52:32 -0500 Subject: [PATCH 19/45] Create synk.yml --- .github/workflows/synk.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .github/workflows/synk.yml diff --git a/.github/workflows/synk.yml b/.github/workflows/synk.yml new file mode 100644 index 00000000..f72c46e1 --- /dev/null +++ b/.github/workflows/synk.yml @@ -0,0 +1,30 @@ +# name: Snyk - Check for vulnerabilities in this Golang project +# on: push +# jobs: +# security: +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@master +# - name: Run Snyk to check for vulnerabilities +# uses: snyk/actions/golang@master +# env: +# SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} + +name: Snyk - Check for vulnerabilities in this Golang project +on: push +jobs: + security: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - name: Run Snyk to check for vulnerabilities + uses: snyk/actions/golang@master + continue-on-error: true # To make sure that SARIF upload gets called + env: + SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} + with: + args: --sarif-file-output=snyk.sarif + - name: Upload result to GitHub Code Scanning + uses: github/codeql-action/upload-sarif@v1 + with: + sarif_file: snyk.sarif From 630103d2aac9ee6e95adf42d9ec9139601a7408b Mon Sep 17 00:00:00 2001 From: repo-visualizer Date: Tue, 5 Apr 2022 21:52:50 +0000 Subject: [PATCH 20/45] Repo visualizer: updated diagram --- repo-visualizer.svg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo-visualizer.svg b/repo-visualizer.svg index faf427f7..18735bfb 100644 --- a/repo-visualizer.svg +++ b/repo-visualizer.svg @@ -1 +1 @@ -srcsrc.github/workflows.github/workflowsmain_test.gomain_test.gomain_test.gomain.gomain.gomain.goREADME.mdREADME.mdREADME.mdLICENSELICENSELICENSEMakefileMakefileMakefiledeployment.ymldeployment.ymldeployment.ymlservice.ymlservice.ymlservice.ymlkustomization...kustomization...kustomization....gitignore.gitignore.gitignoredocker-com...docker-com...docker-com...sonar-pro...sonar-pro...sonar-pro...DockerfileDockerfileDockerfiledeploy.ymldeploy.ymldeploy.ymlmain.ymlmain.ymlmain.ymlquality.ymlquality.ymlquality.ymlmain-test.yml.b...main-test.yml.b...main-test.yml.b...sec.ymlsec.ymlsec.ymlverify.ymlverify.ymlverify.ymlcreate-dia...create-dia...create-dia....gitignore.go.md.mod.properties.svg.ymleach dot sized by file size \ No newline at end of file +srcsrc.github/workflows.github/workflowsmain_test.gomain_test.gomain_test.gomain.gomain.gomain.goREADME.mdREADME.mdREADME.mdLICENSELICENSELICENSEMakefileMakefileMakefiledeployment.ymldeployment.ymldeployment.ymlservice.ymlservice.ymlservice.ymlkustomizatio...kustomizatio...kustomizatio....gitignore.gitignore.gitignoredocker-com...docker-com...docker-com...sonar-pro...sonar-pro...sonar-pro...deploy.ymldeploy.ymldeploy.ymlmain.ymlmain.ymlmain.ymlquality.ymlquality.ymlquality.ymlmain-test.yml.b...main-test.yml.b...main-test.yml.b...synk.ymlsynk.ymlsynk.ymlsec.ymlsec.ymlsec.ymlverify.ymlverify.ymlverify.ymlcreate-dia...create-dia...create-dia....gitignore.go.md.mod.properties.svg.ymleach dot sized by file size \ No newline at end of file From 785b712f55f875374f7b5bfdf08fd59056c0dd1b Mon Sep 17 00:00:00 2001 From: Drew Khoury Date: Tue, 5 Apr 2022 16:55:46 -0500 Subject: [PATCH 21/45] Update synk.yml --- .github/workflows/synk.yml | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/.github/workflows/synk.yml b/.github/workflows/synk.yml index f72c46e1..084b2dd4 100644 --- a/.github/workflows/synk.yml +++ b/.github/workflows/synk.yml @@ -1,16 +1,4 @@ -# name: Snyk - Check for vulnerabilities in this Golang project -# on: push -# jobs: -# security: -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@master -# - name: Run Snyk to check for vulnerabilities -# uses: snyk/actions/golang@master -# env: -# SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} - -name: Snyk - Check for vulnerabilities in this Golang project + name: Snyk - Check for vulnerabilities in this Golang project on: push jobs: security: From b8996021bad6bb8d8e6e4c944986ae244ffbb126 Mon Sep 17 00:00:00 2001 From: repo-visualizer Date: Tue, 5 Apr 2022 21:56:01 +0000 Subject: [PATCH 22/45] Repo visualizer: updated diagram --- repo-visualizer.svg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo-visualizer.svg b/repo-visualizer.svg index 18735bfb..c3852140 100644 --- a/repo-visualizer.svg +++ b/repo-visualizer.svg @@ -1 +1 @@ -srcsrc.github/workflows.github/workflowsmain_test.gomain_test.gomain_test.gomain.gomain.gomain.goREADME.mdREADME.mdREADME.mdLICENSELICENSELICENSEMakefileMakefileMakefiledeployment.ymldeployment.ymldeployment.ymlservice.ymlservice.ymlservice.ymlkustomizatio...kustomizatio...kustomizatio....gitignore.gitignore.gitignoredocker-com...docker-com...docker-com...sonar-pro...sonar-pro...sonar-pro...deploy.ymldeploy.ymldeploy.ymlmain.ymlmain.ymlmain.ymlquality.ymlquality.ymlquality.ymlmain-test.yml.b...main-test.yml.b...main-test.yml.b...synk.ymlsynk.ymlsynk.ymlsec.ymlsec.ymlsec.ymlverify.ymlverify.ymlverify.ymlcreate-dia...create-dia...create-dia....gitignore.go.md.mod.properties.svg.ymleach dot sized by file size \ No newline at end of file +srcsrc.github/workflows.github/workflowsmain_test.gomain_test.gomain_test.gomain.gomain.gomain.goREADME.mdREADME.mdREADME.mdLICENSELICENSELICENSEMakefileMakefileMakefiledeployment.ymldeployment.ymldeployment.ymlservice.ymlservice.ymlservice.ymlkustomizatio...kustomizatio...kustomizatio....gitignore.gitignore.gitignoredocker-com...docker-com...docker-com...sonar-pro...sonar-pro...sonar-pro...deploy.ymldeploy.ymldeploy.ymlmain.ymlmain.ymlmain.ymlquality.ymlquality.ymlquality.ymlmain-test.yml.b...main-test.yml.b...main-test.yml.b...sec.ymlsec.ymlsec.ymlsynk.ymlsynk.ymlsynk.ymlverify.ymlverify.ymlverify.ymlcreate-dia...create-dia...create-dia....gitignore.go.md.mod.properties.svg.ymleach dot sized by file size \ No newline at end of file From bf0ba14ea2a9af7b474593be341a460bf43437c6 Mon Sep 17 00:00:00 2001 From: T903297 Date: Wed, 4 May 2022 11:08:46 -0500 Subject: [PATCH 23/45] Cleanup all phony and add help and list examples --- Makefile | 46 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index c5294026..1f51a521 100644 --- a/Makefile +++ b/Makefile @@ -1,34 +1,56 @@ +3M_IMAGE_NAME ?= flemay/musketeers DOCKER_TAG ?= go-hello-world FULL_TAG ?= ${DOCKER_TAG}:${HASH} DYNAMODB_TABLE ?= ${DOCKER_TAG} PORT ?= "8080" GO_TEST_DOCKER_COMPOSE ?= docker-compose run --rm gobase go test -v -cover AWS_CLI_DOCKER_COMPOSE ?= docker-compose run --rm awscli -HASH := $(shell git rev-parse HEAD) -VERACODE_ID?= "someveracodeid" +HASH := $(shell git rev-parse HEAD) +VERACODE_ID ?= "someveracodeid" +ENVFILE ?= aws.template -ENVFILE ?= aws.template +.DEFAULT_GOAL := help -envfile: +.PHONY: help +help: ## List of targets with descriptions + @echo "\n--------------------- Run [TARGET] [ARGS] or "make help" for more information ---------------------\n" + @for MAKEFILENAME in $(MAKEFILE_LIST); do \ + grep -E '[a-zA-Z_-]+:.*?## .*$$' $$MAKEFILENAME | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'; \ + done + @echo "\n---------------------------------------------------------------------------------------------------\n" + +.PHONY: list +list: pull-3m-image ## Get list of all make targets + @echo "list"; \ + $(DOCKER_RUN_3M) $(MAKE) --no-print-directory _list + +.PHONY: _list +_list: + @echo "\n---------------------------------------\nList of available targets:\n---------------------------------------" + @$(MAKE) -pRrq -f $(firstword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' + @echo "\n---------------------------------------\n" + +.PHONY: envfile +envfile: ## Create envfile echo "from envfile" echo "FOO=${FOO}" echo "BAR=${BAR}" cp $(ENVFILE) aws.env -.PHONY : build -build: +.PHONY: build +build: ## Build the application docker build -t ${FULL_TAG} . .PHONY: run -run: +run: ## Run the application docker run -d -p ${PORT}:${PORT} --name ${DOCKER_TAG} ${FULL_TAG} .PHONY: down -down: +down: ## Stop the application docker rm -f ${DOCKER_TAG} .PHONY: test -test: envfile +test: envfile ## Test the application ${GO_TEST_DOCKER_COMPOSE} .PHONY: create_table @@ -47,6 +69,7 @@ create_table: envfile --provisioned-throughput \ ReadCapacityUnits=10,WriteCapacityUnits=5 +.PHONY: create_tags create_tags: envfile ${AWS_CLI_DOCKER_COMPOSE} dynamodb put-item \ --table-name ${DYNAMODB_TABLE} \ @@ -54,10 +77,13 @@ create_tags: envfile '{"GIT_COMMIT": {"S": "${HASH}"}, "VERACODE_ID":{"S": ${VERACODE_ID}}}' .PHONY: clean -clean: +clean: ## Cleanup and remove docker application docker kill ${DOCKER_TAG} docker rm ${DOCKER_TAG} +.PHONY: pull-3m-image +pull-3m-image: ## Pull 3M image for local executions + docker pull ${3M_IMAGE_NAME} From 047da108abdd0694c9f8c21714ed8fd410026e0d Mon Sep 17 00:00:00 2001 From: Andrew Ochsner Date: Thu, 5 May 2022 09:18:07 -0500 Subject: [PATCH 24/45] fix checks by bumping linter & handling errors --- .github/workflows/quality.yml | 2 +- .github/workflows/synk.yml | 2 +- src/main.go | 5 ++++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 744521f0..fe9137b8 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -15,7 +15,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: lint - uses: golangci/golangci-lint-action@v1 + uses: golangci/golangci-lint-action@v2 with: # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. version: v1.29 diff --git a/.github/workflows/synk.yml b/.github/workflows/synk.yml index 084b2dd4..e5e99a46 100644 --- a/.github/workflows/synk.yml +++ b/.github/workflows/synk.yml @@ -1,4 +1,4 @@ - name: Snyk - Check for vulnerabilities in this Golang project +name: Snyk - Check for vulnerabilities in this Golang project on: push jobs: security: diff --git a/src/main.go b/src/main.go index 595de9d8..fd987e3b 100644 --- a/src/main.go +++ b/src/main.go @@ -41,7 +41,10 @@ func getOneEvent(w http.ResponseWriter, r *http.Request) { for _, singleEvent := range events { if singleEvent.ID == eventID { - json.NewEncoder(w).Encode(singleEvent) + err := json.NewEncoder(w).Encode(singleEvent) + if err != nil { + log.Printf("Error encoding event: %s Error: %s", singleEvent, err) + } } } } From eeda7360f5d0021d91d929ff6d98a8db09a82ab4 Mon Sep 17 00:00:00 2001 From: Andrew Ochsner Date: Thu, 5 May 2022 09:20:29 -0500 Subject: [PATCH 25/45] bump linter one more --- .github/workflows/quality.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index fe9137b8..015113b8 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -15,7 +15,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: lint - uses: golangci/golangci-lint-action@v2 + uses: golangci/golangci-lint-action@v3 with: # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. version: v1.29 From 71e72b3e08dc7d3979d9fb85da89351b3dae4e9c Mon Sep 17 00:00:00 2001 From: repo-visualizer Date: Thu, 5 May 2022 15:03:19 +0000 Subject: [PATCH 26/45] Repo visualizer: updated diagram --- repo-visualizer.svg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo-visualizer.svg b/repo-visualizer.svg index c3852140..9cc08771 100644 --- a/repo-visualizer.svg +++ b/repo-visualizer.svg @@ -1 +1 @@ -srcsrc.github/workflows.github/workflowsmain_test.gomain_test.gomain_test.gomain.gomain.gomain.goREADME.mdREADME.mdREADME.mdLICENSELICENSELICENSEMakefileMakefileMakefiledeployment.ymldeployment.ymldeployment.ymlservice.ymlservice.ymlservice.ymlkustomizatio...kustomizatio...kustomizatio....gitignore.gitignore.gitignoredocker-com...docker-com...docker-com...sonar-pro...sonar-pro...sonar-pro...deploy.ymldeploy.ymldeploy.ymlmain.ymlmain.ymlmain.ymlquality.ymlquality.ymlquality.ymlmain-test.yml.b...main-test.yml.b...main-test.yml.b...sec.ymlsec.ymlsec.ymlsynk.ymlsynk.ymlsynk.ymlverify.ymlverify.ymlverify.ymlcreate-dia...create-dia...create-dia....gitignore.go.md.mod.properties.svg.ymleach dot sized by file size \ No newline at end of file +srcsrc.github/workflows.github/workflowsmain_test.gomain_test.gomain_test.gomain.gomain.gomain.goREADME.mdREADME.mdREADME.mdLICENSELICENSELICENSEMakefileMakefileMakefiledeployment.ymldeployment.ymldeployment.ymlservice.ymlservice.ymlservice.ymlkustomizatio...kustomizatio...kustomizatio....gitignore.gitignore.gitignoredocker-com...docker-com...docker-com...sonar-pro...sonar-pro...sonar-pro...deploy.ymldeploy.ymldeploy.ymlmain.ymlmain.ymlmain.ymlquality.ymlquality.ymlquality.ymlmain-test.yml.b...main-test.yml.b...main-test.yml.b...sec.ymlsec.ymlsec.ymlsynk.ymlsynk.ymlsynk.ymlverify.ymlverify.ymlverify.ymlcreate-dia...create-dia...create-dia....gitignore.go.md.mod.properties.svg.ymleach dot sized by file size \ No newline at end of file From 93cabd5ecfdf35b0a3d42cea6c9734bb2b59c1fe Mon Sep 17 00:00:00 2001 From: repo-visualizer Date: Thu, 5 May 2022 15:03:41 +0000 Subject: [PATCH 27/45] Repo visualizer: updated diagram --- repo-visualizer.svg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo-visualizer.svg b/repo-visualizer.svg index 9cc08771..f1d72439 100644 --- a/repo-visualizer.svg +++ b/repo-visualizer.svg @@ -1 +1 @@ -srcsrc.github/workflows.github/workflowsmain_test.gomain_test.gomain_test.gomain.gomain.gomain.goREADME.mdREADME.mdREADME.mdLICENSELICENSELICENSEMakefileMakefileMakefiledeployment.ymldeployment.ymldeployment.ymlservice.ymlservice.ymlservice.ymlkustomizatio...kustomizatio...kustomizatio....gitignore.gitignore.gitignoredocker-com...docker-com...docker-com...sonar-pro...sonar-pro...sonar-pro...deploy.ymldeploy.ymldeploy.ymlmain.ymlmain.ymlmain.ymlquality.ymlquality.ymlquality.ymlmain-test.yml.b...main-test.yml.b...main-test.yml.b...sec.ymlsec.ymlsec.ymlsynk.ymlsynk.ymlsynk.ymlverify.ymlverify.ymlverify.ymlcreate-dia...create-dia...create-dia....gitignore.go.md.mod.properties.svg.ymleach dot sized by file size \ No newline at end of file +srcsrc.github/workflows.github/workflowsmain_test.gomain_test.gomain_test.gomain.gomain.gomain.goREADME.mdREADME.mdREADME.mdLICENSELICENSELICENSEMakefileMakefileMakefiledeployment.ymldeployment.ymldeployment.ymlservice.ymlservice.ymlservice.ymlkustomizatio...kustomizatio...kustomizatio....gitignore.gitignore.gitignoredocker-com...docker-com...docker-com...sonar-pro...sonar-pro...sonar-pro...deploy.ymldeploy.ymldeploy.ymlmain.ymlmain.ymlmain.ymlquality.ymlquality.ymlquality.ymlmain-test.yml.b...main-test.yml.b...main-test.yml.b...sec.ymlsec.ymlsec.ymlsynk.ymlsynk.ymlsynk.ymlverify.ymlverify.ymlverify.ymlcreate-dia...create-dia...create-dia....gitignore.go.md.mod.properties.svg.ymleach dot sized by file size \ No newline at end of file From 2ac946ebcbebec2ea10f9ddeb91df8580c5f003f Mon Sep 17 00:00:00 2001 From: Paul Kordes Date: Mon, 9 May 2022 09:57:17 -0500 Subject: [PATCH 28/45] Fixpipeline (#20) * Add push to Makefile and fix workflow to use it --- .github/workflows/main.yml | 49 ++++++------------------------------ .github/workflows/synk.yml | 1 + .github/workflows/verify.yml | 1 + Makefile | 24 ++++++++++++------ 4 files changed, 27 insertions(+), 48 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a49a1ab4..1f25e4e5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,10 +13,6 @@ on: # Run tests for any PRs. pull_request: -env: - IMAGE_NAME: go-hello-world - GITHUB_TOKEN: ${{ secrets.GITHUBTOKEN }} # not sure why gh didn't let me create a secret with an underscore in it???????? - jobs: build: @@ -37,16 +33,6 @@ jobs: - name: Run the Go package locally (detached) run: make run - # only used to initially create the table - need a cleaner way to include this, - # and have it do nothing if the table exists - # - # - name: Create DynamboDB Table in AWS - To store metadata (one-time) - # run: make create_table - # env: - # AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # AWS_DEFAULT_REGION: us-east-2 - - name: Create tags in DynamboDB Table in AWS - Metadata for this commit run: make create_tags env: @@ -55,31 +41,12 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_DEFAULT_REGION: us-east-2 - # this was just in here to prove our the publish, should actually edit the make build do write the correct image name then we can remove this step - - name: Build image - run: docker build . --file Dockerfile --tag $IMAGE_NAME - - # this is a bit verbose so probably time we wrapped it in make (it was just a copy/paste from github actions anyway) - - name: Log into registry and Push image - run: | - echo "${{ secrets.GH_PACKAGES_2 }}" | docker login docker.pkg.github.com -u ${{ github.actor }} --password-stdin - - IMAGE_ID=docker.pkg.github.com/${{ github.repository }}/$IMAGE_NAME - - # Change all uppercase to lowercase - IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') - - # Strip git ref prefix from version - VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') - - # Strip "v" prefix from tag name - [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//') - - # Use Docker `latest` tag convention - [ "$VERSION" == "master" ] && VERSION=latest - - echo IMAGE_ID=$IMAGE_ID - echo VERSION=$VERSION + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - docker tag $IMAGE_NAME $IMAGE_ID:$VERSION - docker push $IMAGE_ID:$VERSION + - name: Push image + run: make push diff --git a/.github/workflows/synk.yml b/.github/workflows/synk.yml index e5e99a46..421b73ff 100644 --- a/.github/workflows/synk.yml +++ b/.github/workflows/synk.yml @@ -16,3 +16,4 @@ jobs: uses: github/codeql-action/upload-sarif@v1 with: sarif_file: snyk.sarif + diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 10c93738..0d7977e8 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -21,3 +21,4 @@ jobs: - name: Run all verification rules run: cd gsd-verification-rules && make verify + diff --git a/Makefile b/Makefile index e611cecf..0e1c38e3 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,9 @@ 3M_IMAGE_NAME ?= flemay/musketeers -DOCKER_TAG ?= go-hello-world -FULL_TAG ?= ${DOCKER_TAG}:${HASH} -DYNAMODB_TABLE ?= ${DOCKER_TAG}-v2 +REGISTRY_URL := ghcr.io +GITHUB_REPOSITORY ?= contino/gsd-hello-world +IMAGE_NAME ?= go-hello-world +FULL_TAG ?= ${REGISTRY_URL}/${GITHUB_REPOSITORY}/${IMAGE_NAME}:${HASH} +DYNAMODB_TABLE ?= ${IMAGE_NAME}-v2 PORT ?= "8080" GO_TEST_DOCKER_COMPOSE ?= docker-compose run --rm gobase go test -v -cover AWS_CLI_DOCKER_COMPOSE ?= docker-compose run --rm awscli @@ -41,13 +43,21 @@ envfile: ## Create envfile build: ## Build the application docker build -t ${FULL_TAG} . +.PHONY: push +push: dockerlogin ## Push the containerized application + docker push ${FULL_TAG} + +.PHONY: dockerlogin +dockerlogin: ## Login to docker registry + docker login ${REGISTRY_URL} + .PHONY: run run: ## Run the application - docker run -d -p ${PORT}:${PORT} --name ${DOCKER_TAG} ${FULL_TAG} + docker run -d -p ${PORT}:${PORT} --name ${IMAGE_NAME} ${FULL_TAG} .PHONY: down down: ## Stop the application - docker rm -f ${DOCKER_TAG} + docker rm -f ${IMAGE_NAME} .PHONY: test test: envfile ## Test the application @@ -77,8 +87,8 @@ create_tags: envfile .PHONY: clean clean: ## Cleanup and remove docker application - docker kill ${DOCKER_TAG} - docker rm ${DOCKER_TAG} + docker kill ${IMAGE_NAME} + docker rm ${IMAGE_NAME} .PHONY: pull-3m-image pull-3m-image: ## Pull 3M image for local executions From 772539ddca5c7ae3548f6c2efe591e352d6b1afc Mon Sep 17 00:00:00 2001 From: repo-visualizer Date: Mon, 9 May 2022 14:57:37 +0000 Subject: [PATCH 29/45] Repo visualizer: updated diagram --- repo-visualizer.svg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo-visualizer.svg b/repo-visualizer.svg index f1d72439..985376d2 100644 --- a/repo-visualizer.svg +++ b/repo-visualizer.svg @@ -1 +1 @@ -srcsrc.github/workflows.github/workflowsmain_test.gomain_test.gomain_test.gomain.gomain.gomain.goREADME.mdREADME.mdREADME.mdLICENSELICENSELICENSEMakefileMakefileMakefiledeployment.ymldeployment.ymldeployment.ymlservice.ymlservice.ymlservice.ymlkustomizatio...kustomizatio...kustomizatio....gitignore.gitignore.gitignoredocker-com...docker-com...docker-com...sonar-pro...sonar-pro...sonar-pro...deploy.ymldeploy.ymldeploy.ymlmain.ymlmain.ymlmain.ymlquality.ymlquality.ymlquality.ymlmain-test.yml.b...main-test.yml.b...main-test.yml.b...sec.ymlsec.ymlsec.ymlsynk.ymlsynk.ymlsynk.ymlverify.ymlverify.ymlverify.ymlcreate-dia...create-dia...create-dia....gitignore.go.md.mod.properties.svg.ymleach dot sized by file size \ No newline at end of file +srcsrc.github/workflows.github/workflowsmain_test.gomain_test.gomain_test.gomain.gomain.gomain.goREADME.mdREADME.mdREADME.mdLICENSELICENSELICENSEMakefileMakefileMakefiledeployment.ymldeployment.ymldeployment.ymlservice.ymlservice.ymlservice.ymlkustomization...kustomization...kustomization....gitignore.gitignore.gitignoredocker-com...docker-com...docker-com...sonar-pro...sonar-pro...sonar-pro...DockerfileDockerfileDockerfiledeploy.ymldeploy.ymldeploy.ymlquality.ymlquality.ymlquality.ymlmain.ymlmain.ymlmain.ymlmain-test.yml.b...main-test.yml.b...main-test.yml.b...sec.ymlsec.ymlsec.ymlsynk.ymlsynk.ymlsynk.ymlverify.ymlverify.ymlverify.ymlcreate-dia...create-dia...create-dia....gitignore.go.md.mod.properties.svg.ymleach dot sized by file size \ No newline at end of file From bb719804527715a9d43c2b529bab6044f16ed7a4 Mon Sep 17 00:00:00 2001 From: Paul Kordes Date: Fri, 13 May 2022 09:17:25 -0500 Subject: [PATCH 30/45] Add shell alternative to make for cases when Make isn't available (#21) Added a ./run script that can be used identically to Make in cases where Make may not be available. Fully tested. --- Makefile | 3 +- run | 121 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+), 1 deletion(-) create mode 100755 run diff --git a/Makefile b/Makefile index 0e1c38e3..c698d38d 100644 --- a/Makefile +++ b/Makefile @@ -77,7 +77,8 @@ create_table: envfile --table-name ${DYNAMODB_TABLE} \ --attribute-definitions AttributeName=GIT_COMMIT,AttributeType=S AttributeName=PIPELINE_ID,AttributeType=S \ --key-schema AttributeName=GIT_COMMIT,KeyType=HASH AttributeName=PIPELINE_ID,KeyType=RANGE \ - --provisioned-throughput ReadCapacityUnits=10,WriteCapacityUnits=5 + --provisioned-throughput ReadCapacityUnits=10,WriteCapacityUnits=5 \ + --tags Key=Permanent,Value=True .PHONY: create_tags create_tags: envfile diff --git a/run b/run new file mode 100755 index 00000000..b705e5bb --- /dev/null +++ b/run @@ -0,0 +1,121 @@ +#!/usr/bin/env bash + +# This idea is heavily inspired by: https://github.com/adriancooney/Taskfile +# FOR WHEN YOU DO NOT WANT TO (OR CANNOT) US MAKE +# Found this here.... +# https://www.youtube.com/watch?v=SdmYd5hJISM + +# Ensure specific failures (add u if you want fail on any missing or unset variable) +set -eo pipefail + +# If we're running in CI we need to disable TTY allocation for docker-compose +# commands that enable it by default, such as exec and run. +TTY="" +if [[ ! -t 1 ]]; then + TTY="-T" +fi + +HASH="$(git rev-parse HEAD)" + +: "${ENVFILE:=aws.template}" +: "${IMAGE_NAME_3M:=flemay/musketeers}" +: "${REGISTRY_URL:=ghcr.io}" +: "${GITHUB_REPOSITORY:=contino/gsd-hello-world}" +: "${IMAGE_NAME:=go-hello-world}" +: "${FULL_TAG:=${REGISTRY_URL}/${GITHUB_REPOSITORY}/${IMAGE_NAME}:${HASH}}" +: "${DYNAMODB_TABLE:=${IMAGE_NAME}-v2}" +: "${PORT:=8080}" +: "${PIPELINE_BASE:=contino/gsd-hello-world}" + +# ----------------------------------------------------------------------------- +# Helper functions start with _ and aren't listed in this script's help menu. +# ----------------------------------------------------------------------------- + +function _dc { + docker-compose run --rm "${@}" +} + +# ----------------------------------------------------------------------------- + +function envfile { ## Create envfile + echo "from envfile" + echo "FOO=${FOO}" + echo "BAR=${BAR}" + cp $ENVFILE aws.env +} + +function build { ## Build the application + docker build -t ${FULL_TAG} . +} + +function push { ## Push the containerized application + dockerlogin + docker push ${FULL_TAG} +} + +function dockerlogin { ## Login to docker registry + docker login ${REGISTRY_URL} +} + +function run { ## Run the application + docker run -d -p ${PORT}:${PORT} --name ${IMAGE_NAME} ${FULL_TAG} +} + +function down { ## Stop the application + docker rm -f ${IMAGE_NAME} +} + +function test { ## Test the application + envfile + _dc gobase go test -v -cover +} + +function verify { + git clone git@github.com:contino/gsd-verification-rules.git || true + cd gsd-verification-rules && git pull && make verify +} + +function create_table { + envfile + echo "from create_table" + echo "FOO=${FOO}" + echo "BAR=${BAR}" + _dc awscli dynamodb create-table \ + --table-name ${DYNAMODB_TABLE} \ + --attribute-definitions AttributeName=GIT_COMMIT,AttributeType=S AttributeName=PIPELINE_ID,AttributeType=S \ + --key-schema AttributeName=GIT_COMMIT,KeyType=HASH AttributeName=PIPELINE_ID,KeyType=RANGE \ + --provisioned-throughput ReadCapacityUnits=10,WriteCapacityUnits=5 +} + +function create_tags { + envfile + _dc awscli dynamodb put-item \ + --table-name ${DYNAMODB_TABLE} \ + --item '{ "GIT_COMMIT": {"S": "${HASH}"}, "PIPELINE_BASE":{"S": "${PIPELINE_BASE}"}, "PIPELINE_ID":{"S": "${PIPELINE_ID}"} }' +} + +function clean { ## Cleanup and remove docker application + docker kill ${IMAGE_NAME} + docker rm ${IMAGE_NAME} +} + +function pull-3m-image { ## Pull 3M image for local executions + docker pull ${3M_IMAGE_NAME} +} + +function help { + echo -e "\n------------------------------------- Tasks with descriptions -------------------------------------\n" + grep -E 'function [a-zA-Z_-]+ {.*?## .*$$' ./run | sort | sed 's|function ||' | awk 'BEGIN {FS = " {.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $1, $2}' + echo -e "\n---------------------------------------------------------------------------------------------------\n" +} + +function list { + printf "%s [args]\n\nTasks:\n" "${0}" + + compgen -A function | grep -v "^_" | cat -n + + printf "\nExtended help:\n Each task has comments for general usage\n" +} + +TIMEFORMAT=$'\nTask completed in %3lR' +time "${@:-list}" \ No newline at end of file From 71e47115ee5b960d9fa431e27bf5e4dcf27edfca Mon Sep 17 00:00:00 2001 From: repo-visualizer Date: Fri, 13 May 2022 14:17:42 +0000 Subject: [PATCH 31/45] Repo visualizer: updated diagram --- repo-visualizer.svg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo-visualizer.svg b/repo-visualizer.svg index 985376d2..00b47acf 100644 --- a/repo-visualizer.svg +++ b/repo-visualizer.svg @@ -1 +1 @@ -srcsrc.github/workflows.github/workflowsmain_test.gomain_test.gomain_test.gomain.gomain.gomain.goREADME.mdREADME.mdREADME.mdLICENSELICENSELICENSEMakefileMakefileMakefiledeployment.ymldeployment.ymldeployment.ymlservice.ymlservice.ymlservice.ymlkustomization...kustomization...kustomization....gitignore.gitignore.gitignoredocker-com...docker-com...docker-com...sonar-pro...sonar-pro...sonar-pro...DockerfileDockerfileDockerfiledeploy.ymldeploy.ymldeploy.ymlquality.ymlquality.ymlquality.ymlmain.ymlmain.ymlmain.ymlmain-test.yml.b...main-test.yml.b...main-test.yml.b...sec.ymlsec.ymlsec.ymlsynk.ymlsynk.ymlsynk.ymlverify.ymlverify.ymlverify.ymlcreate-dia...create-dia...create-dia....gitignore.go.md.mod.properties.svg.ymleach dot sized by file size \ No newline at end of file +srcsrc.github/workflows.github/workflowsmain_test.gomain_test.gomain_test.gomain.gomain.gomain.goREADME.mdREADME.mdREADME.mdLICENSELICENSELICENSErunrunrunMakefileMakefileMakefiledeployment.ymldeployment.ymldeployment.ymlservice.ymlservice.ymlservice.ymlkustomization...kustomization...kustomization....gitignore.gitignore.gitignoredocker-com...docker-com...docker-com...sonar-pro...sonar-pro...sonar-pro...DockerfileDockerfileDockerfiledeploy.ymldeploy.ymldeploy.ymlquality.ymlquality.ymlquality.ymlmain.ymlmain.ymlmain.ymlmain-test.yml.b...main-test.yml.b...main-test.yml.b...sec.ymlsec.ymlsec.ymlsynk.ymlsynk.ymlsynk.ymlverify.ymlverify.ymlverify.ymlcreate-dia...create-dia...create-dia....gitignore.go.md.mod.properties.svg.ymleach dot sized by file size \ No newline at end of file From 82f986d7fc623f570a0eb240391c384cd0485457 Mon Sep 17 00:00:00 2001 From: T903297 Date: Mon, 16 May 2022 13:26:18 -0500 Subject: [PATCH 32/45] Force main run --- run | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run b/run index b705e5bb..a067f382 100755 --- a/run +++ b/run @@ -9,7 +9,7 @@ set -eo pipefail # If we're running in CI we need to disable TTY allocation for docker-compose -# commands that enable it by default, such as exec and run. +# commands that enable it by default, such as exec and run. TTY="" if [[ ! -t 1 ]]; then TTY="-T" From d769bfba85905d650b8bf632c740f04fd19cf705 Mon Sep 17 00:00:00 2001 From: repo-visualizer Date: Mon, 16 May 2022 18:26:40 +0000 Subject: [PATCH 33/45] Repo visualizer: updated diagram --- repo-visualizer.svg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo-visualizer.svg b/repo-visualizer.svg index 00b47acf..a1938381 100644 --- a/repo-visualizer.svg +++ b/repo-visualizer.svg @@ -1 +1 @@ -srcsrc.github/workflows.github/workflowsmain_test.gomain_test.gomain_test.gomain.gomain.gomain.goREADME.mdREADME.mdREADME.mdLICENSELICENSELICENSErunrunrunMakefileMakefileMakefiledeployment.ymldeployment.ymldeployment.ymlservice.ymlservice.ymlservice.ymlkustomization...kustomization...kustomization....gitignore.gitignore.gitignoredocker-com...docker-com...docker-com...sonar-pro...sonar-pro...sonar-pro...DockerfileDockerfileDockerfiledeploy.ymldeploy.ymldeploy.ymlquality.ymlquality.ymlquality.ymlmain.ymlmain.ymlmain.ymlmain-test.yml.b...main-test.yml.b...main-test.yml.b...sec.ymlsec.ymlsec.ymlsynk.ymlsynk.ymlsynk.ymlverify.ymlverify.ymlverify.ymlcreate-dia...create-dia...create-dia....gitignore.go.md.mod.properties.svg.ymleach dot sized by file size \ No newline at end of file +srcsrc.github/workflows.github/workflowsmain_test.gomain_test.gomain_test.gomain.gomain.gomain.goREADME.mdREADME.mdREADME.mdLICENSELICENSELICENSErunrunrunMakefileMakefileMakefiledeployment.ymldeployment.ymldeployment.ymlservice.ymlservice.ymlservice.ymlkustomization...kustomization...kustomization....gitignore.gitignore.gitignoredocker-com...docker-com...docker-com...sonar-pro...sonar-pro...sonar-pro...DockerfileDockerfileDockerfiledeploy.ymldeploy.ymldeploy.ymlquality.ymlquality.ymlquality.ymlmain.ymlmain.ymlmain.ymlmain-test.yml.b...main-test.yml.b...main-test.yml.b...sec.ymlsec.ymlsec.ymlsynk.ymlsynk.ymlsynk.ymlverify.ymlverify.ymlverify.ymlcreate-dia...create-dia...create-dia....gitignore.go.md.mod.properties.svg.ymleach dot sized by file size \ No newline at end of file From 67989fc818864c521bbb35de5b0e7b807bd7d0ed Mon Sep 17 00:00:00 2001 From: Drew Khoury Date: Mon, 16 May 2022 16:05:22 -0500 Subject: [PATCH 34/45] allow manual deploy --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 15761151..3e648ace 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -19,7 +19,7 @@ on: push: branches: - main - + workflow_dispatch: env: PROJECT_ID: ${{ secrets.GKE_PROJECT }} From 659f2c9a2255e9b813077bd9bf7199ed1b08736d Mon Sep 17 00:00:00 2001 From: repo-visualizer Date: Mon, 16 May 2022 21:05:38 +0000 Subject: [PATCH 35/45] Repo visualizer: updated diagram --- repo-visualizer.svg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo-visualizer.svg b/repo-visualizer.svg index a1938381..a5f13530 100644 --- a/repo-visualizer.svg +++ b/repo-visualizer.svg @@ -1 +1 @@ -srcsrc.github/workflows.github/workflowsmain_test.gomain_test.gomain_test.gomain.gomain.gomain.goREADME.mdREADME.mdREADME.mdLICENSELICENSELICENSErunrunrunMakefileMakefileMakefiledeployment.ymldeployment.ymldeployment.ymlservice.ymlservice.ymlservice.ymlkustomization...kustomization...kustomization....gitignore.gitignore.gitignoredocker-com...docker-com...docker-com...sonar-pro...sonar-pro...sonar-pro...DockerfileDockerfileDockerfiledeploy.ymldeploy.ymldeploy.ymlquality.ymlquality.ymlquality.ymlmain.ymlmain.ymlmain.ymlmain-test.yml.b...main-test.yml.b...main-test.yml.b...sec.ymlsec.ymlsec.ymlsynk.ymlsynk.ymlsynk.ymlverify.ymlverify.ymlverify.ymlcreate-dia...create-dia...create-dia....gitignore.go.md.mod.properties.svg.ymleach dot sized by file size \ No newline at end of file +srcsrc.github/workflows.github/workflowsmain_test.gomain_test.gomain_test.gomain.gomain.gomain.goREADME.mdREADME.mdREADME.mdLICENSELICENSELICENSErunrunrunMakefileMakefileMakefiledeployment.ymldeployment.ymldeployment.ymlservice.ymlservice.ymlservice.ymlkustomization...kustomization...kustomization....gitignore.gitignore.gitignoredocker-com...docker-com...docker-com...sonar-pro...sonar-pro...sonar-pro...DockerfileDockerfileDockerfiledeploy.ymldeploy.ymldeploy.ymlquality.ymlquality.ymlquality.ymlmain.ymlmain.ymlmain.ymlmain-test.yml.b...main-test.yml.b...main-test.yml.b...sec.ymlsec.ymlsec.ymlsynk.ymlsynk.ymlsynk.ymlverify.ymlverify.ymlverify.ymlcreate-dia...create-dia...create-dia....gitignore.go.md.mod.properties.svg.ymleach dot sized by file size \ No newline at end of file From efb0c90fa512aaf269c94f587d921abe5a704a5b Mon Sep 17 00:00:00 2001 From: Drew Khoury Date: Mon, 16 May 2022 16:37:27 -0500 Subject: [PATCH 36/45] Create deploy-2022.yml --- .github/workflows/deploy-2022.yml | 75 +++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 .github/workflows/deploy-2022.yml diff --git a/.github/workflows/deploy-2022.yml b/.github/workflows/deploy-2022.yml new file mode 100644 index 00000000..52520901 --- /dev/null +++ b/.github/workflows/deploy-2022.yml @@ -0,0 +1,75 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Build and Deploy to GKE - 2022 + +on: + push: + branches: + - main + workflow_dispatch: + +env: + PROJECT_ID: ${{ secrets.GKE_PROJECT }} # contini-XXX-de5a + GKE_CLUSTER: gke-test-2022 # Add your cluster name here. + GKE_ZONE: us-central1 # Add your cluster zone here. + DEPLOYMENT_NAME: gke-from-github # Add your deployment name here. + IMAGE: hello-world + +jobs: + setup-build-publish-deploy: + name: Setup, Build, Publish, and Deploy + runs-on: ubuntu-latest + environment: production + + steps: + - name: Checkout + uses: actions/checkout@v3 + + # Setup gcloud CLI + - uses: google-github-actions/setup-gcloud@94337306dda8180d967a56932ceb4ddcf01edae7 + with: + service_account_key: ${{ secrets.GKE_SA_KEY }} + project_id: ${{ secrets.GKE_PROJECT }} + + # Configure Docker to use the gcloud command-line tool as a credential + # helper for authentication + - run: |- + gcloud --quiet auth configure-docker + + # Get the GKE credentials so we can deploy to the cluster + - uses: google-github-actions/get-gke-credentials@fb08709ba27618c31c09e014e1d8364b02e5042e + with: + cluster_name: ${{ env.GKE_CLUSTER }} + location: ${{ env.GKE_ZONE }} + credentials: ${{ secrets.GKE_SA_KEY }} + + # Build the Docker image + - name: Build + run: |- + docker build \ + --tag "gcr.io/$PROJECT_ID/$IMAGE:$GITHUB_SHA" \ + --build-arg GITHUB_SHA="$GITHUB_SHA" \ + --build-arg GITHUB_REF="$GITHUB_REF" \ + . + + # Push the Docker image to Google Container Registry + - name: Publish + run: |- + docker push "gcr.io/$PROJECT_ID/$IMAGE:$GITHUB_SHA" + + # Set up kustomize + - name: Set up Kustomize + run: |- + curl -sfLo kustomize https://github.com/kubernetes-sigs/kustomize/releases/download/v3.1.0/kustomize_3.1.0_linux_amd64 + chmod u+x ./kustomize + + # Deploy the Docker image to the GKE cluster + - name: Deploy + run: |- + ./kustomize edit set image gcr.io/PROJECT_ID/IMAGE:TAG=gcr.io/$PROJECT_ID/$IMAGE:$GITHUB_SHA + ./kustomize build . | kubectl apply -f - + kubectl rollout status deployment/$DEPLOYMENT_NAME + kubectl get services -o wide From 8e63c5e951fb858cc6901d4e86d9ba911772f017 Mon Sep 17 00:00:00 2001 From: repo-visualizer Date: Mon, 16 May 2022 21:37:44 +0000 Subject: [PATCH 37/45] Repo visualizer: updated diagram --- repo-visualizer.svg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo-visualizer.svg b/repo-visualizer.svg index a5f13530..2439ca05 100644 --- a/repo-visualizer.svg +++ b/repo-visualizer.svg @@ -1 +1 @@ -srcsrc.github/workflows.github/workflowsmain_test.gomain_test.gomain_test.gomain.gomain.gomain.goREADME.mdREADME.mdREADME.mdLICENSELICENSELICENSErunrunrunMakefileMakefileMakefiledeployment.ymldeployment.ymldeployment.ymlservice.ymlservice.ymlservice.ymlkustomization...kustomization...kustomization....gitignore.gitignore.gitignoredocker-com...docker-com...docker-com...sonar-pro...sonar-pro...sonar-pro...DockerfileDockerfileDockerfiledeploy.ymldeploy.ymldeploy.ymlquality.ymlquality.ymlquality.ymlmain.ymlmain.ymlmain.ymlmain-test.yml.b...main-test.yml.b...main-test.yml.b...sec.ymlsec.ymlsec.ymlsynk.ymlsynk.ymlsynk.ymlverify.ymlverify.ymlverify.ymlcreate-dia...create-dia...create-dia....gitignore.go.md.mod.properties.svg.ymleach dot sized by file size \ No newline at end of file +srcsrc.github/workflows.github/workflowsmain_test.gomain_test.gomain_test.gomain.gomain.gomain.goREADME.mdREADME.mdREADME.mdLICENSELICENSELICENSErunrunrunMakefileMakefileMakefiledeployment.ymldeployment.ymldeployment.ymlservice.ymlservice.ymlservice.ymlkustomizatio...kustomizatio...kustomizatio....gitignore.gitignore.gitignoredocker-com...docker-com...docker-com...sonar-pro...sonar-pro...sonar-pro...deploy.ymldeploy.ymldeploy.ymldeploy-2022.ymldeploy-2022.ymldeploy-2022.ymlquality.ymlquality.ymlquality.ymlmain.ymlmain.ymlmain.ymlmain-test.yml....main-test.yml....main-test.yml....sec.ymlsec.ymlsec.ymlsynk.ymlsynk.ymlsynk.ymlverify.ymlverify.ymlverify.ymlcreate-dia...create-dia...create-dia....gitignore.go.md.mod.properties.svg.ymleach dot sized by file size \ No newline at end of file From 5aeab2cf94481a3429697aaa328517e3e512179f Mon Sep 17 00:00:00 2001 From: Drew Khoury Date: Tue, 17 May 2022 13:52:50 -0500 Subject: [PATCH 38/45] gke (#23) * updated k8s config --- .github/workflows/deploy-2022.yml | 75 ------------------------------- .github/workflows/deploy.yml | 61 +++++++++---------------- deployment.yml | 39 +++++++++++----- kustomization.yml | 1 - service.yml | 25 ----------- 5 files changed, 47 insertions(+), 154 deletions(-) delete mode 100644 .github/workflows/deploy-2022.yml delete mode 100644 service.yml diff --git a/.github/workflows/deploy-2022.yml b/.github/workflows/deploy-2022.yml deleted file mode 100644 index 52520901..00000000 --- a/.github/workflows/deploy-2022.yml +++ /dev/null @@ -1,75 +0,0 @@ -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. - -name: Build and Deploy to GKE - 2022 - -on: - push: - branches: - - main - workflow_dispatch: - -env: - PROJECT_ID: ${{ secrets.GKE_PROJECT }} # contini-XXX-de5a - GKE_CLUSTER: gke-test-2022 # Add your cluster name here. - GKE_ZONE: us-central1 # Add your cluster zone here. - DEPLOYMENT_NAME: gke-from-github # Add your deployment name here. - IMAGE: hello-world - -jobs: - setup-build-publish-deploy: - name: Setup, Build, Publish, and Deploy - runs-on: ubuntu-latest - environment: production - - steps: - - name: Checkout - uses: actions/checkout@v3 - - # Setup gcloud CLI - - uses: google-github-actions/setup-gcloud@94337306dda8180d967a56932ceb4ddcf01edae7 - with: - service_account_key: ${{ secrets.GKE_SA_KEY }} - project_id: ${{ secrets.GKE_PROJECT }} - - # Configure Docker to use the gcloud command-line tool as a credential - # helper for authentication - - run: |- - gcloud --quiet auth configure-docker - - # Get the GKE credentials so we can deploy to the cluster - - uses: google-github-actions/get-gke-credentials@fb08709ba27618c31c09e014e1d8364b02e5042e - with: - cluster_name: ${{ env.GKE_CLUSTER }} - location: ${{ env.GKE_ZONE }} - credentials: ${{ secrets.GKE_SA_KEY }} - - # Build the Docker image - - name: Build - run: |- - docker build \ - --tag "gcr.io/$PROJECT_ID/$IMAGE:$GITHUB_SHA" \ - --build-arg GITHUB_SHA="$GITHUB_SHA" \ - --build-arg GITHUB_REF="$GITHUB_REF" \ - . - - # Push the Docker image to Google Container Registry - - name: Publish - run: |- - docker push "gcr.io/$PROJECT_ID/$IMAGE:$GITHUB_SHA" - - # Set up kustomize - - name: Set up Kustomize - run: |- - curl -sfLo kustomize https://github.com/kubernetes-sigs/kustomize/releases/download/v3.1.0/kustomize_3.1.0_linux_amd64 - chmod u+x ./kustomize - - # Deploy the Docker image to the GKE cluster - - name: Deploy - run: |- - ./kustomize edit set image gcr.io/PROJECT_ID/IMAGE:TAG=gcr.io/$PROJECT_ID/$IMAGE:$GITHUB_SHA - ./kustomize build . | kubectl apply -f - - kubectl rollout status deployment/$DEPLOYMENT_NAME - kubectl get services -o wide diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 3e648ace..d8703934 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,44 +1,35 @@ -# This workflow will build a docker container, publish it to Google Container Registry, and deploy it to GKE when a release is created -# -# To configure this workflow: -# -# 1. Ensure that your repository contains the necessary configuration for your Google Kubernetes Engine cluster, including deployment.yml, kustomization.yml, service.yml, etc. -# -# 2. Set up secrets in your workspace: GKE_PROJECT with the name of the project and GKE_SA_KEY with the Base64 encoded JSON service account key (https://github.com/GoogleCloudPlatform/github-actions/tree/docs/service-account-key/setup-gcloud#inputs). -# -# 3. Change the values for the GKE_ZONE, GKE_CLUSTER, IMAGE, and DEPLOYMENT_NAME environment variables (below). -# -# For more support on how to run the workflow, please visit https://github.com/GoogleCloudPlatform/github-actions/tree/master/example-workflows/gke +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. -name: Build and Deploy to GKE +name: Deploy to k8s -#on: -# release: -# types: [created] on: push: branches: - main workflow_dispatch: - + env: - PROJECT_ID: ${{ secrets.GKE_PROJECT }} - GKE_CLUSTER: gsd-stable # using v1.15 (future versions won't allow deployment.yml) https://kubernetes.io/blog/2019/07/18/api-deprecations-in-1-16/ - GKE_ZONE: us-central1-c - DEPLOYMENT_NAME: gke-from-github - IMAGE: hello-world + PROJECT_ID: ${{ secrets.GKE_PROJECT }} # contini-XXX-de5a + GKE_CLUSTER: gke-test-2022 # Add your cluster name here. + GKE_ZONE: us-central1 # Add your cluster zone here. + DEPLOYMENT_NAME: gke-hello-app # Add your deployment name here. + IMAGE: go-hello-world jobs: setup-build-publish-deploy: name: Setup, Build, Publish, and Deploy runs-on: ubuntu-latest + environment: production steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 # Setup gcloud CLI - - uses: GoogleCloudPlatform/github-actions/setup-gcloud@0.1.3 + - uses: google-github-actions/setup-gcloud@94337306dda8180d967a56932ceb4ddcf01edae7 with: service_account_key: ${{ secrets.GKE_SA_KEY }} project_id: ${{ secrets.GKE_PROJECT }} @@ -49,8 +40,11 @@ jobs: gcloud --quiet auth configure-docker # Get the GKE credentials so we can deploy to the cluster - - run: |- - gcloud container clusters get-credentials "$GKE_CLUSTER" --zone "$GKE_ZONE" + - uses: google-github-actions/get-gke-credentials@fb08709ba27618c31c09e014e1d8364b02e5042e + with: + cluster_name: ${{ env.GKE_CLUSTER }} + location: ${{ env.GKE_ZONE }} + credentials: ${{ secrets.GKE_SA_KEY }} # Build the Docker image - name: Build @@ -66,21 +60,6 @@ jobs: run: |- docker push "gcr.io/$PROJECT_ID/$IMAGE:$GITHUB_SHA" - - - name: Build latest - run: |- - docker build \ - --tag "gcr.io/$PROJECT_ID/$IMAGE:latest" \ - --build-arg GITHUB_SHA="$GITHUB_SHA" \ - --build-arg GITHUB_REF="$GITHUB_REF" \ - . - - - name: Publish latest - run: |- - docker push "gcr.io/$PROJECT_ID/$IMAGE:latest" - - - # Set up kustomize - name: Set up Kustomize run: |- @@ -90,7 +69,7 @@ jobs: # Deploy the Docker image to the GKE cluster - name: Deploy run: |- - ./kustomize edit set image gcr.io/PROJECT_ID/IMAGE:TAG=gcr.io/$PROJECT_ID/$IMAGE:$GITHUB_SHA + ./kustomize edit set image gcr.io/$PROJECT_ID/$IMAGE:latest=gcr.io/$PROJECT_ID/$IMAGE:$GITHUB_SHA ./kustomize build . | kubectl apply -f - kubectl rollout status deployment/$DEPLOYMENT_NAME kubectl get services -o wide diff --git a/deployment.yml b/deployment.yml index 399434d5..2ea2eb57 100644 --- a/deployment.yml +++ b/deployment.yml @@ -1,4 +1,4 @@ -# Copyright 2019 Google LLC +# Copyright 2021 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,12 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -apiVersion: apps/v1beta1 +apiVersion: apps/v1 kind: Deployment metadata: - name: gke-test + name: gke-hello-app spec: - replicas: 1 + replicas: 2 + selector: + matchLabels: + app: gke-hello-app strategy: rollingUpdate: maxSurge: 1 @@ -26,15 +29,27 @@ spec: template: metadata: labels: - app: gke-test + app: gke-hello-app spec: containers: - - name: gke-test - image: gcr.io/contino-919ebf1dd6da5f73/hello-world:latest + - name: hello-app + image: gcr.io/contini-0fd0de1002f7de5a/go-hello-world:latest ports: - containerPort: 8080 - resources: - requests: - cpu: 100m - limits: - cpu: 100m +# resources: +# requests: +# cpu: 100m +# limits: +# cpu: 100m +--- +apiVersion: v1 +kind: Service +metadata: + name: gke-hello-app-service +spec: + type: LoadBalancer + ports: + - port: 80 + targetPort: 8080 + selector: + app: gke-hello-app diff --git a/kustomization.yml b/kustomization.yml index 989a38cf..a580c34b 100644 --- a/kustomization.yml +++ b/kustomization.yml @@ -16,4 +16,3 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - deployment.yml -- service.yml diff --git a/service.yml b/service.yml deleted file mode 100644 index 7d6da5b1..00000000 --- a/service.yml +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Service -metadata: - name: gke-test-service -spec: - type: LoadBalancer - ports: - - port: 80 - targetPort: 8080 - selector: - app: gke-test From 9d1feaa64a1db2ce0cb540027c353ce67a078be9 Mon Sep 17 00:00:00 2001 From: repo-visualizer Date: Tue, 17 May 2022 18:53:09 +0000 Subject: [PATCH 39/45] Repo visualizer: updated diagram --- repo-visualizer.svg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo-visualizer.svg b/repo-visualizer.svg index 2439ca05..e951b3f7 100644 --- a/repo-visualizer.svg +++ b/repo-visualizer.svg @@ -1 +1 @@ -srcsrc.github/workflows.github/workflowsmain_test.gomain_test.gomain_test.gomain.gomain.gomain.goREADME.mdREADME.mdREADME.mdLICENSELICENSELICENSErunrunrunMakefileMakefileMakefiledeployment.ymldeployment.ymldeployment.ymlservice.ymlservice.ymlservice.ymlkustomizatio...kustomizatio...kustomizatio....gitignore.gitignore.gitignoredocker-com...docker-com...docker-com...sonar-pro...sonar-pro...sonar-pro...deploy.ymldeploy.ymldeploy.ymldeploy-2022.ymldeploy-2022.ymldeploy-2022.ymlquality.ymlquality.ymlquality.ymlmain.ymlmain.ymlmain.ymlmain-test.yml....main-test.yml....main-test.yml....sec.ymlsec.ymlsec.ymlsynk.ymlsynk.ymlsynk.ymlverify.ymlverify.ymlverify.ymlcreate-dia...create-dia...create-dia....gitignore.go.md.mod.properties.svg.ymleach dot sized by file size \ No newline at end of file +srcsrc.github/workflows.github/workflowsmain_test.gomain_test.gomain_test.gomain.gomain.gomain.goREADME.mdREADME.mdREADME.mdLICENSELICENSELICENSErunrunrunMakefileMakefileMakefiledeployment.ymldeployment.ymldeployment.ymlkustomization...kustomization...kustomization....gitignore.gitignore.gitignoredocker-com...docker-com...docker-com...sonar-pro...sonar-pro...sonar-pro...DockerfileDockerfileDockerfiledeploy.ymldeploy.ymldeploy.ymlquality.ymlquality.ymlquality.ymlmain.ymlmain.ymlmain.ymlmain-test.yml.b...main-test.yml.b...main-test.yml.b...sec.ymlsec.ymlsec.ymlsynk.ymlsynk.ymlsynk.ymlverify.ymlverify.ymlverify.ymlcreate-dia...create-dia...create-dia....gitignore.go.md.mod.properties.svg.ymleach dot sized by file size \ No newline at end of file From a6acd4da67e97589dd6e9ccc876566f0c3f6c5dd Mon Sep 17 00:00:00 2001 From: Jason Lutz <92879381+jasonlutz-contino@users.noreply.github.com> Date: Tue, 17 May 2022 14:42:33 -0700 Subject: [PATCH 40/45] Zap Security Scan in the pipeline (#6) Implements the ZAP security tool to scan and endpoint and produce a report --- .github/workflows/main.yml | 24 +++++++++++++++++-- .github/workflows/{sec.yml => playground.yml} | 5 ++-- .github/workflows/zap-security.yml | 24 +++++++++++++++++++ .gitignore | 3 ++- Makefile | 8 ++++++- aws.template | 1 + docker-compose.yml | 15 +++++++++++- 7 files changed, 73 insertions(+), 7 deletions(-) rename .github/workflows/{sec.yml => playground.yml} (86%) create mode 100644 .github/workflows/zap-security.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1f25e4e5..66af96ed 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,13 +12,14 @@ on: # Run tests for any PRs. pull_request: + types: [opened, synchronize, reopened] jobs: build: runs-on: ubuntu-latest - if: github.event_name == 'push' +# if: github.event_name == 'push' steps: - name: checkout code @@ -26,13 +27,32 @@ jobs: - name: Tests run: make test - + + - name: Security Tests + run: make security + + - name: Archive security results + uses: actions/upload-artifact@v2 + with: + name: security-report + path: output/security-report.txt + - name: Build the Go package run: make build - name: Run the Go package locally (detached) run: make run +# This is only used to initially create the table - need a cleaner way to include this, +# and have it do nothing if the table exists + +# - name: Create DynamboDB Table in AWS - To store metadata (one-time) +# run: make create_table +# env: +# AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} +# AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} +# AWS_DEFAULT_REGION: us-east-2 + - name: Create tags in DynamboDB Table in AWS - Metadata for this commit run: make create_tags env: diff --git a/.github/workflows/sec.yml b/.github/workflows/playground.yml similarity index 86% rename from .github/workflows/sec.yml rename to .github/workflows/playground.yml index f1683efd..f36e864f 100644 --- a/.github/workflows/sec.yml +++ b/.github/workflows/playground.yml @@ -1,10 +1,11 @@ -name: Security +# These are pipeline steps we're testing that aren't GSD-like yet +name: Playground on: push: # Publish `master` as Docker `latest` image. branches: - - master + - main # Publish `v1.2.3` tags as releases. tags: diff --git a/.github/workflows/zap-security.yml b/.github/workflows/zap-security.yml new file mode 100644 index 00000000..a8838363 --- /dev/null +++ b/.github/workflows/zap-security.yml @@ -0,0 +1,24 @@ +on: + push: + branches: + - main + pull_request: + types: [opened, synchronize, reopened] +name: Zap Security Scan + +jobs: + security: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + with: + # Disabling shallow clone is recommended for improving relevancy of reporting + fetch-depth: 0 + - name: security + run: make security + - name: Archive security results + uses: actions/upload-artifact@v2 + with: + name: security-report + path: output/security-report.txt diff --git a/.gitignore b/.gitignore index 8cca8034..4ad3912a 100644 --- a/.gitignore +++ b/.gitignore @@ -21,4 +21,5 @@ aws.env Makefile # gsd specific -gsd-verification-rules \ No newline at end of file +gsd-verification-rules +output/* \ No newline at end of file diff --git a/Makefile b/Makefile index c698d38d..210ea109 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ GITHUB_REPOSITORY ?= contino/gsd-hello-world IMAGE_NAME ?= go-hello-world FULL_TAG ?= ${REGISTRY_URL}/${GITHUB_REPOSITORY}/${IMAGE_NAME}:${HASH} DYNAMODB_TABLE ?= ${IMAGE_NAME}-v2 -PORT ?= "8080" +PORT ?= 8080 GO_TEST_DOCKER_COMPOSE ?= docker-compose run --rm gobase go test -v -cover AWS_CLI_DOCKER_COMPOSE ?= docker-compose run --rm awscli HASH := $(shell git rev-parse HEAD) @@ -68,6 +68,12 @@ verify: git clone git@github.com:contino/gsd-verification-rules.git || true cd gsd-verification-rules && git pull && make verify +.PHONY: security +security: envfile ## Run security checks against app + mkdir -p output + docker-compose run --rm security zap-baseline.py -t http://gohelloworld:${PORT} > output/security-report.txt || true + docker-compose down + .PHONY: create_table create_table: envfile echo "from create_table" diff --git a/aws.template b/aws.template index 2e6b841a..fd7d8bbe 100644 --- a/aws.template +++ b/aws.template @@ -2,4 +2,5 @@ FOO BAR AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY +AWS_SESSION_TOKEN AWS_DEFAULT_REGION diff --git a/docker-compose.yml b/docker-compose.yml index ab618c22..d311d20b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,6 +5,12 @@ services: image: gohelloworld:latest ports: - "8080:8080" + healthcheck: + test: curl --fail http://localhost:8080 || exit 1 + interval: 10s + retries: 50 + start_period: 20s + timeout: 2s gobase: image: golang:latest working_dir: "/app/src" @@ -14,4 +20,11 @@ services: image: amazon/aws-cli working_dir: "/app" env_file: - - ./aws.env \ No newline at end of file + - ./aws.env + security: + image: owasp/zap2docker-weekly + volumes: + - "./output:/output" + depends_on: + gohelloworld: + condition: service_healthy From 1e0e614a021c7708bfd5bb48d2fb697f65186c0c Mon Sep 17 00:00:00 2001 From: repo-visualizer Date: Tue, 17 May 2022 21:42:50 +0000 Subject: [PATCH 41/45] Repo visualizer: updated diagram --- repo-visualizer.svg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo-visualizer.svg b/repo-visualizer.svg index e951b3f7..8fc7bc3a 100644 --- a/repo-visualizer.svg +++ b/repo-visualizer.svg @@ -1 +1 @@ -srcsrc.github/workflows.github/workflowsmain_test.gomain_test.gomain_test.gomain.gomain.gomain.goREADME.mdREADME.mdREADME.mdLICENSELICENSELICENSErunrunrunMakefileMakefileMakefiledeployment.ymldeployment.ymldeployment.ymlkustomization...kustomization...kustomization....gitignore.gitignore.gitignoredocker-com...docker-com...docker-com...sonar-pro...sonar-pro...sonar-pro...DockerfileDockerfileDockerfiledeploy.ymldeploy.ymldeploy.ymlquality.ymlquality.ymlquality.ymlmain.ymlmain.ymlmain.ymlmain-test.yml.b...main-test.yml.b...main-test.yml.b...sec.ymlsec.ymlsec.ymlsynk.ymlsynk.ymlsynk.ymlverify.ymlverify.ymlverify.ymlcreate-dia...create-dia...create-dia....gitignore.go.md.mod.properties.svg.ymleach dot sized by file size \ No newline at end of file +srcsrc.github/workflows.github/workflowsmain_test.gomain_test.gomain_test.gomain.gomain.gomain.goREADME.mdREADME.mdREADME.mdLICENSELICENSELICENSErunrunrunMakefileMakefileMakefiledeployment.ymldeployment.ymldeployment.ymldocker-compos...docker-compos...docker-compos...kustomizatio...kustomizatio...kustomizatio....gitignore.gitignore.gitignoresonar-pro...sonar-pro...sonar-pro...DockerfileDockerfileDockerfiledeploy.ymldeploy.ymldeploy.ymlmain.ymlmain.ymlmain.ymlquality.ymlquality.ymlquality.ymlmain-test.yml.b...main-test.yml.b...main-test.yml.b...playground.ymlplayground.ymlplayground.ymlsynk.ymlsynk.ymlsynk.ymlzap-security...zap-security...zap-security...verify.ymlverify.ymlverify.ymlcreate-dia...create-dia...create-dia....gitignore.go.md.mod.properties.svg.ymleach dot sized by file size \ No newline at end of file From f56020aca05b5e99703561ba8796485f2ed6d219 Mon Sep 17 00:00:00 2001 From: Drew Khoury Date: Fri, 20 May 2022 11:33:06 -0500 Subject: [PATCH 42/45] Deploy docs (#24) docs for deploy --- docs/deploy.md | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 docs/deploy.md diff --git a/docs/deploy.md b/docs/deploy.md new file mode 100644 index 00000000..76f312af --- /dev/null +++ b/docs/deploy.md @@ -0,0 +1,38 @@ +# Deployments + +A deployment takes a built artifact (ie golang app in a docker container) and deploys it to an environment (a k8s cluster). + +## The k8s Environment + +We're using GKE from Google Cloud in our example, but any k8s cluster will work. + +There are currently a few things we're doing manually to prepare an environment: + +- Cluster Creation (Done through Console - Autopilot) +- IAM Service Account using permissions described here [Deploy to k8s](https://docs.github.com/en/actions/deployment/deploying-to-your-cloud-provider/deploying-to-google-kubernetes-engine) - ie cluster admin, and storage admin (you may have to tweak permissions for your own deployment) +- `GKE_SA_KEY` in github populated using the json key created from the above service account - ie `cat key.json | base64` + +The Cluster `gke-test-2022` lives in tihe project `contini-XXX-de5a` which is `Squad Zero > Andrew Khoury Contino`. + +## The deployment + +Our github workflow `deploy.yml` ("Deploy to k8s") does the deployment, and is based on [google-github-actions](https://github.com/google-github-actions/setup-gcloud/tree/main/example-workflows/gke). + +What it does: + +- Sets some key variables `PROJECT_ID`, `GKE_CLUSTER`, `GKE_ZONE`, `DEPLOYMENT_NAME`, `IMAGE` +- Runs as a production deployment, checks out code, auths to gcloud +- builds the app, publishes it +- deploys to k8s after ensuring we're using the newly built image (using a combo of kustomize and kubectl) + +What it depends on: + +- `kustomization.yml` (to tell it to look for `deployment.yml`) +- `deployment.yml` (k8s config - app/service name, container, ports, lb etc) + +# Todos + +- Use dev for this pipeline? And Prod for the prod pipeline? +- Automate env setup +- Make this step gsd like, ie `make deploy` +- Instead of building during this step, leverage the build from the main CI? (build once deploy many) From a9769a848849cebc31a987e5f3ad44136ce0cc21 Mon Sep 17 00:00:00 2001 From: repo-visualizer Date: Fri, 20 May 2022 16:33:22 +0000 Subject: [PATCH 43/45] Repo visualizer: updated diagram --- repo-visualizer.svg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo-visualizer.svg b/repo-visualizer.svg index 8fc7bc3a..5c53ea7a 100644 --- a/repo-visualizer.svg +++ b/repo-visualizer.svg @@ -1 +1 @@ -srcsrc.github/workflows.github/workflowsmain_test.gomain_test.gomain_test.gomain.gomain.gomain.goREADME.mdREADME.mdREADME.mdLICENSELICENSELICENSErunrunrunMakefileMakefileMakefiledeployment.ymldeployment.ymldeployment.ymldocker-compos...docker-compos...docker-compos...kustomizatio...kustomizatio...kustomizatio....gitignore.gitignore.gitignoresonar-pro...sonar-pro...sonar-pro...DockerfileDockerfileDockerfiledeploy.ymldeploy.ymldeploy.ymlmain.ymlmain.ymlmain.ymlquality.ymlquality.ymlquality.ymlmain-test.yml.b...main-test.yml.b...main-test.yml.b...playground.ymlplayground.ymlplayground.ymlsynk.ymlsynk.ymlsynk.ymlzap-security...zap-security...zap-security...verify.ymlverify.ymlverify.ymlcreate-dia...create-dia...create-dia....gitignore.go.md.mod.properties.svg.ymleach dot sized by file size \ No newline at end of file +srcsrc.github/workflows.github/workflowsmain_test.gomain_test.gomain_test.gomain.gomain.gomain.goREADME.mdREADME.mdREADME.mdLICENSELICENSELICENSEdocs/deploy.mddocs/deploy.mddocs/deploy.mdrunrunrunMakefileMakefileMakefiledeployment.ymldeployment.ymldeployment.ymldocker-compo...docker-compo...docker-compo...kustomizatio...kustomizatio...kustomizatio....gitignore.gitignore.gitignoresonar-pro...sonar-pro...sonar-pro...deploy.ymldeploy.ymldeploy.ymlmain.ymlmain.ymlmain.ymlquality.ymlquality.ymlquality.ymlmain-test.yml....main-test.yml....main-test.yml....playground.ymlplayground.ymlplayground.ymlsynk.ymlsynk.ymlsynk.ymlzap-securit...zap-securit...zap-securit...verify.ymlverify.ymlverify.ymlcreate-dia...create-dia...create-dia....gitignore.go.md.mod.properties.svg.ymleach dot sized by file size \ No newline at end of file From df8aa69f9ac95c1e99c0ef8a47aa62317ed35742 Mon Sep 17 00:00:00 2001 From: Paul Kordes Date: Fri, 20 May 2022 13:08:16 -0500 Subject: [PATCH 44/45] Add missing Env variables (#22) Massive changes to get verification fixed * Add missing Env variables * Fix Github artifact token * switch runs-on container to one with jq * Make verify a reusable workflow * Fix race condition that main must finish before verify in order to write tags * Move security scan into main * Remove PIPELINE_ID from table definition as it becomes required when getting items --- .github/workflows/main.yml | 27 ++++++++++-------- .github/workflows/verify.yml | 44 +++++++++++++++++++++++++----- .github/workflows/zap-security.yml | 24 ---------------- .gitignore | 2 +- Makefile | 31 ++++++++++----------- docker-compose.yml | 23 ++++++++++------ security-compose.yml | 22 +++++++++++++++ 7 files changed, 104 insertions(+), 69 deletions(-) delete mode 100644 .github/workflows/zap-security.yml create mode 100644 security-compose.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 66af96ed..cef16f81 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -28,6 +28,9 @@ jobs: - name: Tests run: make test + - name: Build the Go package + run: make build + - name: Security Tests run: make security @@ -35,23 +38,20 @@ jobs: uses: actions/upload-artifact@v2 with: name: security-report - path: output/security-report.txt + path: security-report - - name: Build the Go package - run: make build - name: Run the Go package locally (detached) run: make run # This is only used to initially create the table - need a cleaner way to include this, -# and have it do nothing if the table exists - -# - name: Create DynamboDB Table in AWS - To store metadata (one-time) -# run: make create_table -# env: -# AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} -# AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} -# AWS_DEFAULT_REGION: us-east-2 +# and have it do nothing if the table exists. Make currently ignores error + - name: Create DynamboDB Table in AWS - To store metadata (one-time) + run: make create_table + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: us-east-2 - name: Create tags in DynamboDB Table in AWS - Metadata for this commit run: make create_tags @@ -70,3 +70,8 @@ jobs: - name: Push image run: make push + + call-verify: + uses: ./.github/workflows/verify.yml + needs: [build] + secrets: inherit diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 0d7977e8..b9871e2e 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -1,15 +1,41 @@ on: - push: - branches: - - main - pull_request: - types: [opened, synchronize, reopened] + workflow_call: + secrets: + AWS_ACCESS_KEY_ID: + required: true + AWS_SECRET_ACCESS_KEY: + required: true + GH_ARTIFACT_TOKEN: + required: true + DYNAMODB_TABLE: + required: true + name: Verify +env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + TOKEN: ${{ secrets.GH_ARTIFACT_TOKEN }} + AWS_DEFAULT_REGION: us-east-2 jobs: verify: runs-on: ubuntu-latest steps: + - run: | + echo "🎉 The commit ID of the artifact is: $GITCOMMIT" + echo "GITCOMMIT=$GITCOMMIT" >> $GITHUB_ENV + echo "🎉 The dynamodb table is: $DYNAMODB_TABLE" + echo "DYNAMODB_TABLE=$DYNAMODB_TABLE" >> $GITHUB_ENV + env: + GITCOMMIT: ${{ github.sha }} + DYNAMODB_TABLE: ${{ secrets.DYNAMODB_TABLE }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Check out gsd-verification-rules repo uses: actions/checkout@v2 @@ -18,7 +44,11 @@ jobs: ref: main path: gsd-verification-rules ssh-key: "${{ secrets.SSH_PRIVATE_KEY }}" + + - name: Download all workflow run artifacts + uses: actions/download-artifact@v3 + with: + path: gsd-verification-rules - name: Run all verification rules - run: cd gsd-verification-rules && make verify - + run: cd gsd-verification-rules && env && make verify diff --git a/.github/workflows/zap-security.yml b/.github/workflows/zap-security.yml deleted file mode 100644 index a8838363..00000000 --- a/.github/workflows/zap-security.yml +++ /dev/null @@ -1,24 +0,0 @@ -on: - push: - branches: - - main - pull_request: - types: [opened, synchronize, reopened] -name: Zap Security Scan - -jobs: - security: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - with: - # Disabling shallow clone is recommended for improving relevancy of reporting - fetch-depth: 0 - - name: security - run: make security - - name: Archive security results - uses: actions/upload-artifact@v2 - with: - name: security-report - path: output/security-report.txt diff --git a/.gitignore b/.gitignore index 4ad3912a..a98b8c76 100644 --- a/.gitignore +++ b/.gitignore @@ -22,4 +22,4 @@ Makefile # gsd specific gsd-verification-rules -output/* \ No newline at end of file +security-report/* \ No newline at end of file diff --git a/Makefile b/Makefile index 210ea109..ffb52450 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,7 @@ IMAGE_NAME ?= go-hello-world FULL_TAG ?= ${REGISTRY_URL}/${GITHUB_REPOSITORY}/${IMAGE_NAME}:${HASH} DYNAMODB_TABLE ?= ${IMAGE_NAME}-v2 PORT ?= 8080 +DOCKER_COMPOSE ?= FULL_TAG=${FULL_TAG} docker-compose GO_TEST_DOCKER_COMPOSE ?= docker-compose run --rm gobase go test -v -cover AWS_CLI_DOCKER_COMPOSE ?= docker-compose run --rm awscli HASH := $(shell git rev-parse HEAD) @@ -34,9 +35,6 @@ _list: .PHONY: envfile envfile: ## Create envfile - echo "from envfile" - echo "FOO=${FOO}" - echo "BAR=${BAR}" cp $(ENVFILE) aws.env .PHONY: build @@ -53,11 +51,12 @@ dockerlogin: ## Login to docker registry .PHONY: run run: ## Run the application - docker run -d -p ${PORT}:${PORT} --name ${IMAGE_NAME} ${FULL_TAG} + $(DOCKER_COMPOSE) up -d gohelloworld + $(DOCKER_COMPOSE) up healthcheck .PHONY: down down: ## Stop the application - docker rm -f ${IMAGE_NAME} + $(DOCKER_COMPOSE) down .PHONY: test test: envfile ## Test the application @@ -69,22 +68,21 @@ verify: cd gsd-verification-rules && git pull && make verify .PHONY: security -security: envfile ## Run security checks against app - mkdir -p output - docker-compose run --rm security zap-baseline.py -t http://gohelloworld:${PORT} > output/security-report.txt || true - docker-compose down +security: run ## Run security checks against app + rm -rf security-report + mkdir -p security-report + $(DOCKER_COMPOSE) -p security -f security-compose.yml up + $(DOCKER_COMPOSE) -p security -f security-compose.yml down || true + @$(MAKE) --no-print-directory down .PHONY: create_table create_table: envfile - echo "from create_table" - echo "FOO=${FOO}" - echo "BAR=${BAR}" - ${AWS_CLI_DOCKER_COMPOSE} dynamodb create-table \ + -${AWS_CLI_DOCKER_COMPOSE} dynamodb create-table \ --table-name ${DYNAMODB_TABLE} \ - --attribute-definitions AttributeName=GIT_COMMIT,AttributeType=S AttributeName=PIPELINE_ID,AttributeType=S \ - --key-schema AttributeName=GIT_COMMIT,KeyType=HASH AttributeName=PIPELINE_ID,KeyType=RANGE \ + --attribute-definitions AttributeName=GIT_COMMIT,AttributeType=S \ + --key-schema AttributeName=GIT_COMMIT,KeyType=HASH \ --provisioned-throughput ReadCapacityUnits=10,WriteCapacityUnits=5 \ - --tags Key=Permanent,Value=True + --tags Key=Permanent,Value=True .PHONY: create_tags create_tags: envfile @@ -102,5 +100,4 @@ pull-3m-image: ## Pull 3M image for local executions docker pull ${3M_IMAGE_NAME} - diff --git a/docker-compose.yml b/docker-compose.yml index d311d20b..aa103663 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,8 +1,13 @@ version: '3' + +networks: + default: + external: false + name: go-hello-world + services: gohelloworld: - build: . - image: gohelloworld:latest + image: ${FULL_TAG} ports: - "8080:8080" healthcheck: @@ -11,20 +16,20 @@ services: retries: 50 start_period: 20s timeout: 2s + healthcheck: + image: alpine/curl + command: http://gohelloworld:8080 --max-time 60 --retry-max-time 60 --connect-timeout 5 --retry 10 --retry-connrefused --silent + gobase: image: golang:latest working_dir: "/app/src" volumes: - "./:/app" + profiles: ["test"] awscli: image: amazon/aws-cli working_dir: "/app" env_file: - ./aws.env - security: - image: owasp/zap2docker-weekly - volumes: - - "./output:/output" - depends_on: - gohelloworld: - condition: service_healthy + profiles: ["aws"] + diff --git a/security-compose.yml b/security-compose.yml new file mode 100644 index 00000000..3db10f3c --- /dev/null +++ b/security-compose.yml @@ -0,0 +1,22 @@ +version: '3' + +networks: + default: + external: false + name: go-hello-world + +services: + + zap-scan: + user: root + image: owasp/zap2docker-weekly + volumes: + - "./security-report:/security-report" + command: ["/bin/sh","-c","zap-baseline.py -t http://gohelloworld:8080 > /security-report/zap-security-report.txt || true"] + + # Used for testing but not very good. Waiting to add something better here + # nikto-scan: + # image: sullo/nikto + # volumes: + # - "./security-report:/output" + # command: -h http://gohelloworld:8080 -o output/nikto-security-report.txt From 066885d8e3461ba91913db0b93e2430af003fd8a Mon Sep 17 00:00:00 2001 From: repo-visualizer Date: Fri, 20 May 2022 18:08:34 +0000 Subject: [PATCH 45/45] Repo visualizer: updated diagram --- repo-visualizer.svg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo-visualizer.svg b/repo-visualizer.svg index 5c53ea7a..bd47336d 100644 --- a/repo-visualizer.svg +++ b/repo-visualizer.svg @@ -1 +1 @@ -srcsrc.github/workflows.github/workflowsmain_test.gomain_test.gomain_test.gomain.gomain.gomain.goREADME.mdREADME.mdREADME.mdLICENSELICENSELICENSEdocs/deploy.mddocs/deploy.mddocs/deploy.mdrunrunrunMakefileMakefileMakefiledeployment.ymldeployment.ymldeployment.ymldocker-compo...docker-compo...docker-compo...kustomizatio...kustomizatio...kustomizatio....gitignore.gitignore.gitignoresonar-pro...sonar-pro...sonar-pro...deploy.ymldeploy.ymldeploy.ymlmain.ymlmain.ymlmain.ymlquality.ymlquality.ymlquality.ymlmain-test.yml....main-test.yml....main-test.yml....playground.ymlplayground.ymlplayground.ymlsynk.ymlsynk.ymlsynk.ymlzap-securit...zap-securit...zap-securit...verify.ymlverify.ymlverify.ymlcreate-dia...create-dia...create-dia....gitignore.go.md.mod.properties.svg.ymleach dot sized by file size \ No newline at end of file +srcsrc.github/workflows.github/workflowsmain_test.gomain_test.gomain_test.gomain.gomain.gomain.goREADME.mdREADME.mdREADME.mdLICENSELICENSELICENSEdocs/deploy.mddocs/deploy.mddocs/deploy.mdrunrunrunMakefileMakefileMakefiledeployment.ymldeployment.ymldeployment.ymldocker-compo...docker-compo...docker-compo...kustomizatio...kustomizatio...kustomizatio...security-co...security-co...security-co....gitignore.gitignore.gitignoresonar-pro...sonar-pro...sonar-pro...deploy.ymldeploy.ymldeploy.ymlmain.ymlmain.ymlmain.ymlverify.ymlverify.ymlverify.ymlquality.ymlquality.ymlquality.ymlmain-test.yml....main-test.yml....main-test.yml....playground.ymlplayground.ymlplayground.ymlsynk.ymlsynk.ymlsynk.ymlcreate-dia...create-dia...create-dia....gitignore.go.md.mod.properties.svg.ymleach dot sized by file size \ No newline at end of file