Merge branch 'main' into feature/vote-decrypt

.github/workflows/pick-to-staging.yml

@@ -0,0 +1,66 @@
+name: Cherry pick staging PRs merged into main
+
+on:
+  pull_request_target:
+    types:
+      - closed
+    branches:
+      - 'main'
+
+
+jobs:
+  create-pr-for-staging:
+    if: |
+      github.event.pull_request.merged == true &&
+      contains(github.event.pull_request.labels.*.name, 'staging')
+    name: Create PR against staging branch
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout main
+        uses: actions/checkout@v4
+        with:
+          ref: main
+          fetch-depth: 2
+
+      - name: Fetch and checkout latest staging branch
+        run: |
+          branch=$(git ls-remote --heads origin 'staging/*' | awk 'gsub(".*refs/heads/","")' | sort -V | tail -1)
+          git fetch origin $branch
+          git checkout $branch
+
+      - name: Set git credentials
+        run: |
+          git config --global user.name openslides-automation
+          git config --global user.email [email protected]
+
+      - name: Cherry-pick new commit
+        id: cherry-pick
+        run: |
+          git fetch origin
+          # -m 1 to also be able to cherry-pick merge commits
+          git cherry-pick -m 1 ${{ github.sha }} || {
+            echo "error=1" >> $GITHUB_OUTPUT
+            git add .
+            git cherry-pick --continue
+          }
+
+      - name: Generate access token
+        uses: tibdex/github-app-token@v2
+        id: generate-token
+        with:
+          app_id: ${{ secrets.AUTOMATION_APP_ID }}
+          private_key: ${{ secrets.AUTOMATION_APP_PRIVATE_KEY }}
+
+      - name: Create or update PR
+        uses: peter-evans/create-pull-request@v6
+        with:
+          token: ${{ steps.generate-token.outputs.token }}
+          branch: apply/commit-${{ github.sha }}
+          delete-branch: true
+          title: "[Cherry-Pick] ${{ github.event.pull_request.title }}"
+          body: "Triggered by commit [${{ github.sha }}](https://github.com/${{ github.repository }}/commit/${{ github.sha }})\n\n${{ steps.cherry-pick.outputs.error && 'There were conflicts during the cherry-pick. These were commited without any resolving. Please resolve them manually and push the result to this branch before merging.' || 'The cherry-pick was successful without any conflicts. You should be able to simply merge this PR.' }}"
+          reviewers: ${{ github.event.pull_request.user.login }}
+          assignees: ${{ github.event.pull_request.user.login }}
+          labels: picked-to-staging
+          milestone: 4

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.22.1-alpine as base
+FROM golang:1.22.3-alpine as base
 WORKDIR /root/
 
 RUN apk add git

README.md

@@ -229,36 +229,49 @@ The logged metric is a json dictonary like:
 
 ```json
 {
-    "connected_users_anonymous_connections": 1,
-    "connected_users_average_connections": 3,
-    "connected_users_current": 3,
-    "connected_users_current_local": 3,
-    "connected_users_total": 3,
-    "connected_users_total_local": 3,
-    "current_connections": 8,
-    "current_connections_local": 8,
-    "datastore_cache_key_len": 343,
-    "datastore_cache_size": 3114,
-    "runtime_goroutines": 42
+    "connections_longpolling_connected_users_anonymous_connections": 0,
+    "connections_longpolling_connected_users_average_connections": 3,
+    "connections_longpolling_connected_users_current": 1,
+    "connections_longpolling_connected_users_current_local": 1,
+    "connections_longpolling_connected_users_total": 1,
+    "connections_longpolling_connected_users_total_local": 1,
+    "connections_longpolling_current_connections": 3,
+    "connections_longpolling_current_connections_local": 3,
+    "connections_stream_connected_users_anonymous_connections": 0,
+    "connections_stream_connected_users_average_connections": 6,
+    "connections_stream_connected_users_current": 2,
+    "connections_stream_connected_users_current_local": 2,
+    "connections_stream_connected_users_total": 3,
+    "connections_stream_connected_users_total_local": 3,
+    "connections_stream_current_connections": 13,
+    "connections_stream_current_connections_local": 13,
+    "datastore_cache_key_len": 236478,
+    "datastore_cache_size": 1722987,
+    "runtime_goroutines": 68
 }
 ```
 
-The values are:
-
-* `connected_users_anonymous_connections`: Number of connections from the
-  anonymous users from all autoupdate instances.
-* `connected_users_average_connections`: Average connection count for each user
-  except for anonymous user.
-* `connected_users_current`: Amount of connected users that have at least one
-  open connection.
-* `connected_users_current_local`: Amount of connected users that have at least
-  one open connection of this instance.
-* `connected_users_total`: Amount of different users that are currently
-  connected or were connected since the autoupdate service was started.
-* `connected_users_total_local`: Same as `connected_users_total`, but only for this
-instance.
-* `current_connections`: Amount of all connections.
-* `current_connections_local`: Amount of all connections of this instance.
+The prefix `connections_stream` are for "normal" connections.
+`connections_longpolling` are for connections, that use the longpolling
+fallback.
+
+
+
+* `connections_stream_connected_users_anonymous_connections`: Number of
+  connections from the anonymous users from all autoupdate instances.
+* `connections_stream_connected_users_average_connections`: Average connection
+  count for each user except for anonymous user.
+* `connections_stream_connected_users_current`: Amount of connected users that
+  have at least one open connection.
+* `connections_stream_connected_users_current_local`: Amount of connected users
+  that have at least one open connection of this instance.
+* `connections_stream_connected_users_total`: Amount of different users that are
+  currently connected or were connected since the autoupdate service was
+  started.
+* `connections_stream_connected_users_total_local`: Same as
+`connected_users_total`, but only for this instance.
+* `connections_stream_current_connections`: Amount of all connections.
+* `connections_stream_current_connections_local`: Amount of all connections of this instance.
 * `datastore_cache_key_len`: Amount of keys in the cache.
 * `datastore_cache_size`: Combined size of all values in the cache.
 * `runtime_goroutines`: Current goroutines used by the instance.

go.mod

@@ -7,8 +7,8 @@ require (
 	github.com/goccy/go-yaml v1.11.3
 	github.com/golang-jwt/jwt/v4 v4.5.0
 	github.com/gomodule/redigo v1.9.2
-	github.com/jackc/pgx/v5 v5.5.5
-	github.com/klauspost/compress v1.17.7
+	github.com/jackc/pgx/v5 v5.6.0
+	github.com/klauspost/compress v1.17.8
 	github.com/nsf/jsondiff v0.0.0-20230430225905-43f6cf3098c1
 	github.com/ory/dockertest/v3 v3.10.0
 	github.com/ostcar/topic v0.4.1

go.sum

@@ -57,14 +57,14 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI
 github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
 github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 h1:L0QtFUgDarD7Fpv9jeVMgy/+Ec0mtnmYuImjTz6dtDA=
 github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
-github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw=
-github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A=
+github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY=
+github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw=
 github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
 github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg=
-github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
+github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM=
 github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
 github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=

internal/autoupdate/autoupdate.go

@@ -216,8 +216,8 @@ func (a *Autoupdate) skipWorkpool(ctx context.Context, userID int) (bool, error)
 			return false, fmt.Errorf("checking for admin groups: %w", err)
 		}
 
-		for isAdmin := range adminGroups {
-			if isAdmin > 0 {
+		for _, isAdmin := range adminGroups {
+			if _, ok := isAdmin.Value(); ok {
 				return true, nil
 			}
 		}

internal/http/connection_count.go

@@ -18,25 +18,27 @@ type RedisMetric interface {
 	Get(ctx context.Context) (map[int]int, error)
 }
 
-// connectionCount counts, how many connections a user has.
+// ConnectionCount counts, how many connections a user has.
 //
-// It holds a local counter and saves it to redis after a connection is created
-// or closed.
+// It holds a local counter and saves it to redis from time to time. The
+// argument `saveInterval` defines, how oftem it is saved.
 //
-// It also pings redis from time to time to show, that this instance is
-// still running.
-type connectionCount struct {
+// It also pings redis from time to time to show, that this instance is still
+// running.
+type ConnectionCount struct {
 	metric RedisMetric
+	name   string
 
 	mu          sync.Mutex
 	connections map[int]int
 }
 
-func newConnectionCount(ctx context.Context, r *redis.Redis, saveInterval time.Duration) *connectionCount {
-	redisMetric := redis.NewMetric[map[int]int](r, "autoupdate_connection_count", mapIntCombiner{}, saveInterval*2, time.Now)
+func newConnectionCount(ctx context.Context, r *redis.Redis, saveInterval time.Duration, name string) *ConnectionCount {
+	redisMetric := redis.NewMetric[map[int]int](r, name, mapIntCombiner{}, saveInterval*2, time.Now)
 
-	c := connectionCount{
+	c := ConnectionCount{
 		metric:      redisMetric,
+		name:        name,
 		connections: make(map[int]int),
 	}
 
@@ -60,7 +62,7 @@ func newConnectionCount(ctx context.Context, r *redis.Redis, saveInterval time.D
 	return &c
 }
 
-func (c *connectionCount) save(ctx context.Context) error {
+func (c *ConnectionCount) save(ctx context.Context) error {
 	c.mu.Lock()
 	converted, err := json.Marshal(c.connections)
 	c.mu.Unlock()
@@ -75,31 +77,38 @@ func (c *connectionCount) save(ctx context.Context) error {
 	return nil
 }
 
-func (c *connectionCount) increment(uid int, increment int) {
+func (c *ConnectionCount) increment(uid int, increment int) {
 	c.mu.Lock()
 	c.connections[uid] += increment
 	c.mu.Unlock()
 }
 
-func (c *connectionCount) Add(uid int) {
+// Add adds one connection to the counter.
+func (c *ConnectionCount) Add(uid int) {
 	c.increment(uid, 1)
 }
 
-func (c *connectionCount) Done(uid int) {
+// Done removes one connection from the counter.
+func (c *ConnectionCount) Done(uid int) {
 	c.increment(uid, -1)
 }
 
-func (c *connectionCount) Show(ctx context.Context) (map[int]int, error) {
+// Show shoes the counter.
+func (c *ConnectionCount) Show(ctx context.Context, filter func(ctx context.Context, count map[int]int) error) (map[int]int, error) {
 	data, err := c.metric.Get(ctx)
 	if err != nil {
 		return nil, fmt.Errorf("getting counter from redis: %w", err)
 	}
 
+	if err := filter(ctx, data); err != nil {
+		return nil, fmt.Errorf("filtering counter: %w", err)
+	}
+
 	return data, nil
 }
 
 // Metric is a function needed my the openslides metric system to fetch some values.
-func (c *connectionCount) Metric(con metric.Container) {
+func (c *ConnectionCount) Metric(con metric.Container) {
 	ctx := context.Background()
 
 	data, err := c.metric.Get(ctx)
@@ -145,17 +154,15 @@ func (c *connectionCount) Metric(con metric.Container) {
 		average = averageSum / averageCount
 	}
 
-	prefix := "connected_users"
-	con.Add(prefix+"_current", currentConnectedUsers)
-	con.Add(prefix+"_total", len(data))
-	con.Add(prefix+"_current_local", localCurrentUsers)
-	con.Add(prefix+"_total_local", totalCurrentConnections)
-	con.Add(prefix+"_average_connections", average)
-	con.Add(prefix+"_anonymous_connections", data[0])
-
-	prefix = "current_connections"
-	con.Add(prefix, currentConnections)
-	con.Add(prefix+"_local", localCurrentConnections)
+	con.Add(c.name+"_connected_users_current", currentConnectedUsers)
+	con.Add(c.name+"_connected_users_total", len(data))
+	con.Add(c.name+"_connected_users_current_local", localCurrentUsers)
+	con.Add(c.name+"_connected_users_total_local", totalCurrentConnections)
+	con.Add(c.name+"_connected_users_average_connections", average)
+	con.Add(c.name+"_connected_users_anonymous_connections", data[0])
+
+	con.Add(c.name+"_current_connections", currentConnections)
+	con.Add(c.name+"_current_connections_local", localCurrentConnections)
 }
 
 // mapIntCombiner tells the redis Metric, how to combine the metric values.