feat: implement state storage

On a best-effort basis, store the state on the disk periodically and on shutdown & restore it from the disk on startup. Additionally, bump Go version, deps & rekres. Closes #54. Signed-off-by: Utku Ozdemir <[email protected]>
siderolabs · May 22, 2024 · ea8b8f1 · ea8b8f1
1 parent c551426
commit ea8b8f1
Show file tree

Hide file tree

Showing 24 changed files with 2,928 additions and 150 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -1,8 +1,9 @@
 # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
 #
-# Generated on 2023-01-30T15:45:28Z by kres latest.
+# Generated on 2024-05-09T09:59:51Z by kres 1e986af.
 
 *
+!api
 !cmd
 !internal
 !pkg

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -1,6 +1,6 @@
 # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
 #
-# Generated on 2024-03-12T11:34:19Z by kres latest.
+# Generated on 2024-05-21T10:07:54Z by kres 0290180.
 
 name: default
 concurrency:
@@ -29,27 +29,19 @@ jobs:
       - self-hosted
       - generic
     if: (!startsWith(github.head_ref, 'renovate/') && !startsWith(github.head_ref, 'dependabot/'))
-    services:
-      buildkitd:
-        image: moby/buildkit:v0.12.5
-        options: --privileged
-        ports:
-          - 1234:1234
-        volumes:
-          - /var/lib/buildkit/${{ github.repository }}:/var/lib/buildkit
-          - /usr/etc/buildkit/buildkitd.toml:/etc/buildkit/buildkitd.toml
     steps:
       - name: checkout
         uses: actions/checkout@v4
       - name: Unshallow
         run: |
           git fetch --prune --unshallow
       - name: Set up Docker Buildx
+        id: setup-buildx
         uses: docker/setup-buildx-action@v3
         with:
           driver: remote
-          endpoint: tcp://127.0.0.1:1234
-        timeout-minutes: 1
+          endpoint: tcp://buildkit-amd64.ci.svc.cluster.local:1234
+        timeout-minutes: 10
       - name: base
         run: |
           make base
@@ -60,8 +52,11 @@ jobs:
         run: |
           make unit-tests-race
       - name: coverage
-        run: |
-          make coverage
+        uses: codecov/codecov-action@v4
+        with:
+          files: _out/coverage-unit-tests.txt
+          token: ${{ secrets.CODECOV_TOKEN }}
+        timeout-minutes: 3
       - name: discovery-service
         run: |
           make discovery-service
@@ -86,17 +81,20 @@ jobs:
         run: |
           make image-discovery-service
       - name: push-discovery-service-latest
-        if: github.event_name != 'pull_request'
+        if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main'
         env:
           PLATFORM: linux/amd64,linux/arm64
           PUSH: "true"
         run: |
-          make image-discovery-service TAG=latest
+          make image-discovery-service IMAGE_TAG=latest
+      - name: snapshot-decoder
+        run: |
+          make snapshot-decoder
       - name: Generate Checksums
         if: startsWith(github.ref, 'refs/tags/')
         run: |
-          sha256sum _out/discovery-service-* > _out/sha256sum.txt
-          sha512sum _out/discovery-service-* > _out/sha512sum.txt
+          sha256sum _out/discovery-service-* _out/snapshot-decoder-* > _out/sha256sum.txt
+          sha512sum _out/discovery-service-* _out/snapshot-decoder-* > _out/sha512sum.txt
       - name: release-notes
         if: startsWith(github.ref, 'refs/tags/')
         run: |
@@ -109,4 +107,5 @@ jobs:
           draft: "true"
           files: |-
             _out/discovery-service-*
+            _out/snapshot-decoder-*
             _out/sha*.txt
diff --git a/.golangci.yml b/.golangci.yml
@@ -1,21 +1,20 @@
 # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
 #
-# Generated on 2024-03-12T11:34:19Z by kres latest.
+# Generated on 2024-05-19T19:39:10Z by kres dccd292.
 
 # options for analysis running
 run:
   timeout: 10m
   issues-exit-code: 1
   tests: true
-  build-tags: []
-  skip-dirs: []
-  skip-dirs-use-default: true
-  skip-files: []
+  build-tags: [ ]
   modules-download-mode: readonly
 
 # output configuration options
 output:
-  format: colored-line-number
+  formats:
+  - format: colored-line-number
+    path: stdout
   print-issued-lines: true
   print-linter-name: true
   uniq-by-line: true
@@ -32,54 +31,35 @@ linters-settings:
     check-blank: true
   exhaustive:
     default-signifies-exhaustive: false
-  funlen:
-    lines: 60
-    statements: 40
   gci:
-    local-prefixes: github.com/siderolabs/discovery-service/
+    sections:
+      - standard # Standard section: captures all standard packages.
+      - default # Default section: contains all imports that could not be matched to another section type.
+      - localmodule # Imports from the same module.
   gocognit:
     min-complexity: 30
-  ireturn:
-    allow:
-      - anon
-      - error
-      - empty
-      - stdlib
-      - github.com\/talos-systems\/kres\/internal\/dag.Node
   nestif:
     min-complexity: 5
   goconst:
     min-len: 3
     min-occurrences: 3
   gocritic:
-    disabled-checks: []
+    disabled-checks: [ ]
   gocyclo:
     min-complexity: 20
   godot:
-    check-all: false
-  godox:
-    keywords: # default keywords are TODO, BUG, and FIXME, these can be overwritten by this setting
-      - NOTE
-      - OPTIMIZE # marks code that should be optimized before merging
-      - HACK # marks hack-arounds that should be removed before merging
+    scope: declarations
   gofmt:
     simplify: true
-  goimports:
-    local-prefixes: github.com/siderolabs/discovery-service/
-  golint:
-    min-confidence: 0.8
-  gomnd:
-    settings: {}
-  gomodguard: {}
+  gomodguard: { }
   govet:
-    check-shadowing: true
     enable-all: true
   lll:
     line-length: 200
     tab-width: 4
   misspell:
     locale: US
-    ignore-words: []
+    ignore-words: [ ]
   nakedret:
     max-func-lines: 30
   prealloc:
@@ -88,16 +68,15 @@ linters-settings:
     for-loops: false # Report preallocation suggestions on for loops, false by default
   nolintlint:
     allow-unused: false
-    allow-leading-space: false
-    allow-no-explanation: []
+    allow-no-explanation: [ ]
     require-explanation: false
     require-specific: true
-  rowserrcheck: {}
-  testpackage: {}
+  rowserrcheck: { }
+  testpackage: { }
   unparam:
     check-exported: false
   unused:
-    check-exported: false
+    local-variables-are-used: false
   whitespace:
     multi-if: false   # Enforces newlines (or comments) after every multi-line if statement
     multi-func: false # Enforces newlines (or comments) after every multi-line function signature
@@ -113,8 +92,8 @@ linters-settings:
   gofumpt:
     extra-rules: false
   cyclop:
-      # the maximal code complexity to report
-      max-complexity: 20
+    # the maximal code complexity to report
+    max-complexity: 20
   #  depguard:
   #    Main:
   #      deny:
@@ -125,48 +104,50 @@ linters:
   disable-all: false
   fast: false
   disable:
-    - exhaustruct
     - exhaustivestruct
+    - exhaustruct
+    - err113
     - forbidigo
     - funlen
-    - gas
     - gochecknoglobals
     - gochecknoinits
     - godox
-    - goerr113
     - gomnd
     - gomoddirectives
+    - gosec
+    - inamedparam
     - ireturn
+    - mnd
     - nestif
     - nonamedreturns
     - nosnakecase
     - paralleltest
+    - tagalign
     - tagliatelle
     - thelper
     - typecheck
     - varnamelen
     - wrapcheck
     - depguard # Disabled because starting with golangci-lint 1.53.0 it doesn't allow denylist alone anymore
-    - tagalign
-    - inamedparam
     - testifylint # complains about our assert recorder and has a number of false positives for assert.Greater(t, thing, 1)
     - protogetter # complains about us using Value field on typed spec, instead of GetValue which has a different signature
     - perfsprint # complains about us using fmt.Sprintf in non-performance critical code, updating just kres took too long
     # abandoned linters for which golangci shows the warning that the repo is archived by the owner
+    - deadcode
+    - golint
+    - ifshort
     - interfacer
     - maligned
-    - golint
     - scopelint
-    - varcheck
-    - deadcode
     - structcheck
-    - ifshort
+    - varcheck
     # disabled as it seems to be broken - goes into imported libraries and reports issues there
     - musttag
+    - goimports # same as gci
 
 issues:
-  exclude: []
-  exclude-rules: []
+  exclude: [ ]
+  exclude-rules: [ ]
   exclude-use-default: false
   exclude-case-sensitive: false
   max-issues-per-linter: 10

diff --git a/.kres.yaml b/.kres.yaml
@@ -1,4 +1,9 @@
 ---
+kind: auto.CommandConfig
+name: snapshot-decoder
+spec:
+  disableImage: true
+---
 kind: common.Image
 name: image-discovery-service
 spec:
@@ -15,6 +20,15 @@ spec:
       GOOS: linux
       GOARCH: arm64
 ---
+kind: golang.Generate
+spec:
+  baseSpecPath: /api
+  vtProtobufEnabled: true
+  specs:
+    - source: api/storage/storage.proto
+      subdirectory: storage
+      genGateway: false
+---
 kind: service.CodeCov
 spec:
   targetThreshold: 30