Initial setup (#1)

Initial setup

.env.dist

@@ -0,0 +1,2 @@
+LOG_LEVEL=debug
+GOOGLE_PROJECT_ID=

.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: gomod
+    directory: /
+    schedule:
+      interval: daily

.github/workflows/ci.yml

@@ -0,0 +1,38 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+permissions:
+  contents: read
+
+jobs:
+  linting:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set up Go
+      uses: actions/setup-go@v4
+      with:
+        go-version: '1.21.5'
+        cache: false
+
+    - name: Run linting
+      uses: golangci/golangci-lint-action@v3
+      with:
+        version: latest
+        args: --timeout=5m
+
+  security:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Scan for Vulnerabilities in Code
+        uses: golang/govulncheck-action@v1
+        with:
+          go-version-input: 1.21.5

.github/workflows/release.yml

@@ -0,0 +1,44 @@
+name: release
+
+on:
+  push:
+    # run only against tags
+    tags:
+      - '*'
+
+permissions:
+  contents: write
+  # packages: write
+  # issues: write
+
+jobs:
+  goreleaser:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - run: git fetch --force --tags
+
+      - name: Docker Login
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2      
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - uses: actions/setup-go@v4
+        with:
+          go-version: stable
+      - uses: goreleaser/goreleaser-action@v4
+        with:
+          distribution: goreleaser
+          version: latest
+          args: release --clean
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -0,0 +1,3 @@
+.idea
+.env
+.DS_*

.golangci.yml

@@ -0,0 +1,17 @@
+issues:
+  exclude-rules:
+    - path: (.+)_test.go
+      linters:
+          - errcheck
+          - staticcheck
+
+linters-settings:
+  errcheck:
+    # Report about not checking of errors in type assertions: `a := b.(MyStruct)`.
+    # Such cases aren't reported by default.
+    # Default: false
+    check-type-assertions: true
+    # report about assignment of errors to blank identifier: `num, _ := strconv.Atoi(numStr)`.
+    # Such cases aren't reported by default.
+    # Default: false
+    check-blank: true

.goreleaser.yml

@@ -0,0 +1,110 @@
+before:
+  hooks:
+    - go mod download
+    - go generate ./...
+builds:
+  - env:
+    - CGO_ENABLED=0
+    goos:
+      - linux
+      - darwin
+      - windows
+    goarch:
+      - amd64
+      - arm
+      - arm64
+    goarm:
+      - 6
+      - 7
+    mod_timestamp: "{{ .CommitTimestamp }}"
+archives:
+  - format: tar.gz
+    # this name template makes the OS and Arch compatible with the results of uname.
+    name_template: >-
+      {{ .ProjectName }}_
+      {{- title .Os }}_
+      {{- if eq .Arch "amd64" }}x86_64
+      {{- else if eq .Arch "386" }}i386
+      {{- else }}{{ .Arch }}{{ end }}
+      {{- if .Arm }}v{{ .Arm }}{{ end }}
+    # use zip for windows archives
+    format_overrides:
+    - goos: windows
+      format: zip
+    files:
+      - README.md
+      - LICENSE
+checksum:
+  name_template: 'checksums.txt'
+snapshot:
+  name_template: "{{ .Tag }}-next"
+changelog:
+  sort: asc
+  filters:
+    exclude:
+      - '^docs:'
+      - '^test:'
+dockers:
+  - use: buildx
+    goos: linux
+    goarch: amd64
+    image_templates:
+      - "yoursurprise/gcp-pubsub-deadletter-logger:{{ .Version }}-amd64"
+      - "yoursurprise/gcp-pubsub-deadletter-logger:latest-amd64"
+    build_flag_templates:
+      - "--platform=linux/amd64"
+      - "--label=org.opencontainers.image.created={{.Date}}"
+      - "--label=org.opencontainers.image.title={{.ProjectName}}"
+      - "--label=org.opencontainers.image.revision={{.FullCommit}}"
+      - "--label=org.opencontainers.image.version={{.Version}}"
+  - use: buildx
+    goos: linux
+    goarch: arm64
+    image_templates:
+      - "yoursurprise/gcp-pubsub-deadletter-logger:{{ .Version }}-arm64v8"
+      - "yoursurprise/gcp-pubsub-deadletter-logger:latest-arm64v8"
+    build_flag_templates:
+      - "--platform=linux/arm64/v8"
+      - "--label=org.opencontainers.image.created={{.Date}}"
+      - "--label=org.opencontainers.image.title={{.ProjectName}}"
+      - "--label=org.opencontainers.image.revision={{.FullCommit}}"
+      - "--label=org.opencontainers.image.version={{.Version}}"
+  - use: buildx
+    goos: linux
+    goarch: arm
+    goarm: 6
+    image_templates:
+      - "yoursurprise/gcp-pubsub-deadletter-logger:{{ .Version }}-armv6"
+      - "yoursurprise/gcp-pubsub-deadletter-logger:latest-armv6"
+    build_flag_templates:
+      - "--platform=linux/arm/v6"
+      - "--label=org.opencontainers.image.created={{.Date}}"
+      - "--label=org.opencontainers.image.title={{.ProjectName}}"
+      - "--label=org.opencontainers.image.revision={{.FullCommit}}"
+      - "--label=org.opencontainers.image.version={{.Version}}"
+  - use: buildx
+    goos: linux
+    goarch: arm
+    goarm: 7
+    image_templates:
+      - "yoursurprise/gcp-pubsub-deadletter-logger:{{ .Version }}-armv7"
+      - "yoursurprise/gcp-pubsub-deadletter-logger:latest-armv7"
+    build_flag_templates:
+      - "--platform=linux/arm/v7"
+      - "--label=org.opencontainers.image.created={{.Date}}"
+      - "--label=org.opencontainers.image.title={{.ProjectName}}"
+      - "--label=org.opencontainers.image.revision={{.FullCommit}}"
+      - "--label=org.opencontainers.image.version={{.Version}}"
+docker_manifests:
+  - name_template: yoursurprise/gcp-pubsub-deadletter-logger:{{ .Version }}
+    image_templates:
+      - yoursurprise/gcp-pubsub-deadletter-logger:{{ .Version }}-amd64
+      - yoursurprise/gcp-pubsub-deadletter-logger:{{ .Version }}-arm64v8
+      - yoursurprise/gcp-pubsub-deadletter-logger:{{ .Version }}-armv6
+      - yoursurprise/gcp-pubsub-deadletter-logger:{{ .Version }}-armv7
+  - name_template: yoursurprise/gcp-pubsub-deadletter-logger:latest
+    image_templates:
+      - yoursurprise/gcp-pubsub-deadletter-logger:latest-amd64
+      - yoursurprise/gcp-pubsub-deadletter-logger:latest-arm64v8
+      - yoursurprise/gcp-pubsub-deadletter-logger:latest-armv6
+      - yoursurprise/gcp-pubsub-deadletter-logger:latest-armv7

Dockerfile

@@ -0,0 +1,11 @@
+FROM alpine:latest AS build
+
+RUN apk add --update --no-cache ca-certificates
+
+
+FROM scratch
+
+COPY --from=build  /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+COPY  ./gcp-pubsub-deadletter-logger /gcp-pubsub-deadletter-logger
+
+ENTRYPOINT [ "/gcp-pubsub-deadletter-logger" ]

Makefile

@@ -0,0 +1,44 @@
+.DEFAULT_GOAL := help
+
+.PHONY: help
+help:
+	@printf "\033[33mUsage:\033[0m\n  make [target] [arg=\"val\"...]\n\n\033[33mTargets:\033[0m\n"
+	@grep -E '^[-a-zA-Z0-9_\.\/]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "  \033[32m%-15s\033[0m %s\n", $$1, $$2}'
+
+.PHONY: init
+init: .env  install ## Initialise and install all dependencies
+
+.PHONY: install 
+install: ## install dependencies 
+	@go mod tidy
+	@go mod download
+
+.PHONY: run
+run: .env ## Start the application
+	@bash -c "env `cat .env | xargs` go run ."
+
+.env: | .env.dist
+	@echo "Copying .env.dist to .env"
+	@cp -nv .env.dist .env
+
+.PHONY: check
+check: golangci-lint go-vet unit-tests security-code-scan security-vulnerability-scan ## Run application checks
+.PHONY: golangci-lint
+golangci-lint:
+	@golangci-lint run
+
+.PHONY: go-vat
+go-vet:
+	@go vet
+
+.PHONY: unit-tests
+unit-tests:
+	@go test ./...
+
+.PHONY: security-code-scan
+security-code-scan:
+	@gosec ./...
+
+.PHONY: security-vulnerability-scan
+security-vulnerability-scan:
+	@govulncheck ./...

README.md

@@ -1,2 +1,69 @@
-# gcp-pubsub-deadletter-logger
-Log dead letter events to Google Cloud Logging
+# GCP Pub/Sub dead-letter logger
+[![Go Report Card](https://goreportcard.com/badge/github.com/YourSurpriseCom/gcp-pubsub-deadletter-logger)](https://goreportcard.com/report/github.com/YourSurpriseCom/gcp-pubsub-deadletter-logger) 
+![workflow ci](https://github.com/YourSurpriseCom/gcp-pubsub-deadletter-logger/actions/workflows/ci.yml/badge.svg)
+![workflow release ](https://github.com/YourSurpriseCom/gcp-pubsub-deadletter-logger/actions/workflows/release.yml/badge.svg)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+## Overview
+This application is used to log dead-letter events to Google Cloud Logging. 
+
+### Example usage
+The BigQuery subscription can sent every event directly to a BigQuery table. 
+When an event cannot be added to the BigQuery table, the error is added to the event message and sent to the dead-letter topic.
+Setting this application as an Cloud-Run push subscription on the dead-letter topic, the error and event will be logged to Google Cloud Logging.
+This way its possible to see the error, which occurred on the BigQuery subscription directly in Google Cloud Logging.
+
+## Installation
+This application should be installed as a dead-letter topic push subscription to a Cloud-Run instance running this application.
+
+### Configuration
+Setup a Cloud-Run instance and use the docker container `yoursurprise/gcp-pubsub-deadletter-logger` with the following environment variables:
+
+* `GOOGLE_PROJECT_ID` - holds the Google project id where it should write to.
+* `LOG_LEVEL` - Enable optional debug information (`info` or `debug`)
+
+
+## Communication flow
+To use this service, your topic needs a dead letter topic, which has a push subscription to a cloud run container running this software. See graph below:
+
+```
++--------+        +--------------+            +------------+       +-------------------+       +-----------------+
+| NORMAL |        | PUSH         |  (failed)  | DEADLETTER |       | PUSH Subscription |       |  Cloud Logging  |
+| Topic  |  ==>   | Subscription |     ==>    | Topic      |  ==>  | Deadletter-logger |  ==>  | (event + error) |
++--------+        +--------------+            +------------+       +-------------------+       +-----------------+ 
+```
+
+## Usage
+This application is available as a docker container and can be started as following:
+
+```shell
+foo@bar:~$ docker run --name gcp-pubsub-deadletter-logger \
+    -e GOOGLE_PROJECT_ID="<google-project-id>" \
+    -e LOG_LEVEL="debug" \
+    yoursurprise/gcp-pubsub-deadletter-logger:latest
+```
+
+## Local developing
+* Run `make init` to create the `.env` file and install the required packages.
+* Edit `.env` and change the values where needed.
+* Run `make run` to start the application.
+* Send the following message as a post to this application while running on port `8888`:
+
+```json
+{
+    "message":  {
+        "attributes":{
+            "CloudPubSubDeadLetterSourceDeliveryCount":"1337", 
+            "CloudPubSubDeadLetterSourceDeliveryErrorMessage":"Some error occured, which why this message is now a deadletter", 
+            "CloudPubSubDeadLetterSourceSubscription":"example-subscription", 
+            "CloudPubSubDeadLetterSourceSubscriptionProject":"google-project-id", 
+            "CloudPubSubDeadLetterSourceTopicPublishTime":"2023-12-01T20:30:00.000+00:00" 
+        }, 
+        "data":"eyJmaWVsZDEiIDogImV4YW1wbGUgZGF0YSIsImZpZWxkMiIgOiAxMzM3LCAiZmllbGQzIiA6ICJ0aGlzIG1lc3NhZ2Ugc2hvdWxkIGJlIHJlYWRhYmxlIGluc2lkZSB0aGUgZ29vZ2xlIGNsb3VkIGxvZyJ9", 
+        "messageId":"123456790", 
+        "message_id":"123456790", 
+        "publishTime":"2023-12-01T20:30:00.000Z", 
+        "publish_time":"2023-12-01T20:30:01.000Z"
+    }
+}
+```