feat!: upload inspected file to s3

This commit adds functionality that uploads files to s3 and removes
them instead of copying them to a target directory. Validation of
WARC-files is removed as it should happen later in the pipeline.

.github/workflows/main.yaml

@@ -13,8 +13,7 @@ jobs:
             - name: Setup go
               uses: actions/setup-go@v5
               with:
-                  go-version: "1.21"
-                  cache: false
+                  go-version: "stable"
             - name: Build
               run: "go build ./..."
             - name: Test
@@ -28,13 +27,11 @@ jobs:
             - name: Setup go
               uses: actions/setup-go@v5
               with:
-                  go-version: "1.21"
-                  cache: false
+                  go-version: "stable"
             - name: Lint
-              uses: golangci/golangci-lint-action@v3
+              uses: golangci/golangci-lint-action@v6
               with:
-                  version: v1.55.1
-                  args: --config=.golangci.yaml
+                  version: latest
     codeqL-build:
         name: CodeQL build
         runs-on: ubuntu-latest

.golangci.yaml

@@ -1,5 +1,6 @@
 output:
-  format: github-actions
+  formats:
+    - format: colored-line-number
   print-issued-lines: true
   print-linter-name: true
   uniq-by-line: true
@@ -14,6 +15,7 @@ issues:
   max-same-issues: 0
 
 severity:
+  default-severity: error
   case-sensitive: true
 
 linters:
@@ -27,16 +29,15 @@ linters:
     - bodyclose
     - containedctx
     - cyclop
-    - deadcode
     - decorder
     - depguard
     - dogsled
     - dupl
     - dupword
+    - err113
     - errchkjson
     - execinquery
     - exhaustive
-    - exhaustivestruct
     - exhaustruct
     - exportloopref
     - forbidigo
@@ -53,30 +54,26 @@ linters:
     - gocyclo
     - godot
     - godox
-    - goerr113
     - gofumpt
     - goheader
     - goimports
-    - golint
     - gomnd
     - gomoddirectives
     - gomodguard
     - goprintffuncname
     - gosec
     - gosmopolitan
     - grouper
-    - ifshort
     - importas
     - interfacebloat
-    - interfacer
     - ireturn
     - lll
     - loggercheck
     - maintidx
     - makezero
-    - maligned
     - mirror
     - misspell
+    - mnd
     - musttag
     - nakedret
     - nestif
@@ -86,7 +83,6 @@ linters:
     - noctx
     - nolintlint
     - nonamedreturns
-    - nosnakecase
     - nosprintfhostport
     - paralleltest
     - prealloc
@@ -95,9 +91,7 @@ linters:
     - reassign
     - revive
     - rowserrcheck
-    - scopelint
     - sqlclosecheck
-    - structcheck
     - stylecheck
     - tagalign
     - tagliatelle
@@ -109,7 +103,6 @@ linters:
     - unconvert
     - unparam
     - usestdlibvars
-    - varcheck
     - varnamelen
     - wastedassign
     - whitespace

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.21 AS build
+FROM golang:1.23 AS build
 
 WORKDIR /go/src/app
 

go.mod

@@ -1,27 +1,50 @@
 module github.com/nlnwa/fai
 
-go 1.21
+go 1.22.0
+
+toolchain go1.22.7
 
 require (
-	github.com/nlnwa/gowarc v1.1.1
-	github.com/prometheus/client_golang v1.17.0
+	github.com/minio/minio-go/v7 v7.0.77
+	github.com/prometheus/client_golang v1.20.4
+	github.com/spf13/pflag v1.0.5
+	github.com/spf13/viper v1.19.0
 )
 
 require (
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/bits-and-blooms/bitset v1.10.0 // indirect
-	github.com/cespare/xxhash/v2 v2.2.0 // indirect
-	github.com/google/uuid v1.4.0 // indirect
-	github.com/klauspost/compress v1.17.2 // indirect
-	github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
-	github.com/nlnwa/whatwg-url v0.4.0 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
-	github.com/prometheus/client_model v0.5.0 // indirect
-	github.com/prometheus/common v0.45.0 // indirect
-	github.com/prometheus/procfs v0.12.0 // indirect
-	github.com/prometheus/prometheus v0.47.2 // indirect
-	golang.org/x/net v0.17.0 // indirect
-	golang.org/x/sys v0.13.0 // indirect
-	golang.org/x/text v0.13.0 // indirect
-	google.golang.org/protobuf v1.31.0 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/go-ini/ini v1.67.0 // indirect
+	github.com/goccy/go-json v0.10.3 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/klauspost/compress v1.17.10 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.8 // indirect
+	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/minio/md5-simd v1.1.2 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
+	github.com/prometheus/client_model v0.6.1 // indirect
+	github.com/prometheus/common v0.60.0 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
+	github.com/rs/xid v1.6.0 // indirect
+	github.com/sagikazarmark/locafero v0.6.0 // indirect
+	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+	github.com/sourcegraph/conc v0.3.0 // indirect
+	github.com/spf13/afero v1.11.0 // indirect
+	github.com/spf13/cast v1.7.0 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
+	go.uber.org/atomic v1.11.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	golang.org/x/crypto v0.28.0 // indirect
+	golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c // indirect
+	golang.org/x/net v0.30.0 // indirect
+	golang.org/x/sys v0.26.0 // indirect
+	golang.org/x/text v0.19.0 // indirect
+	google.golang.org/protobuf v1.35.1 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )