release v0.0.4: queue topics and module API

chore: set version to 0.0.4.dev

chore: convert CRLF to LF in .gitignore

refactor: localstack -> dynamodb-local for demo

refactor: move tests out of src/

test: fix some linter bugs

refactor: hide the ‘brrr’ singleton

It leaks API and the name clash with the module is extremely confusing.

chore: remove unused asyncrrr.py

test: run linter

.gitignore

@@ -1,7 +1,10 @@
-.venv
-.envrc
-.terraform*
-data
-__pycache__
-result
-.direnv
+.venv
+.envrc
+.terraform*
+data
+__pycache__
+result
+.direnv
+
+# Noise from dynamodb-local program in the working directory
+dynamodb-local-metadata.json

README.md

@@ -56,14 +56,9 @@ Drawback: the call graph must be idempotent, meaning: for the same inputs, a tas
 
 ## Demo
 
-### Requirements
+Requires [Nix](https://nixos.org), with flakes enabled.
 
-- Docker
-- [Nix](https://nixos.org)
-
-### Instructions
-
-Make sure you have Docker running, and start the full demo:
+You can start the full demo without installation:
 
 ```
 $ nix run github:nobssoftware/brrr#demo

brrr-demo.nix → brrr-demo.service.nix

@@ -12,7 +12,9 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-# Brrr-demo module for process-compose-flake
+# Brrr-demo module for services-flake.  Awkwardly named file because
+# services-flake insists on auto-deriving the module name from the filename.
+# Ok.
 
 { config, pkgs, name, lib, ... }: {
   options = with lib.types; {

brrr_demo.py

@@ -11,7 +11,7 @@
 
 from brrr.backends import redis as redis_, dynamo
 import brrr
-from brrr import task, wrrrk, setup, brrr
+from brrr import task
 
 @bottle.route("/<task_name>")
 def get_or_schedule_task(task_name: str):
@@ -33,9 +33,6 @@ def get_or_schedule_task(task_name: str):
         return {"status": "accepted"}
 
 def init_brrr(reset_backends):
-    # Check credentials
-    boto3.client('sts').get_caller_identity()
-
     redis_client = redis.Redis(decode_responses=True)
     queue = redis_.RedisStream(redis_client, os.environ.get("REDIS_QUEUE_KEY", "r1"))
     if reset_backends:
@@ -46,14 +43,15 @@ def init_brrr(reset_backends):
     if reset_backends:
         store.create_table()
 
-    setup(queue, store)
+    brrr.setup(queue, store)
 
 @task
 def fib(n: int, salt=None):
     match n:
-        case 0: return 0
-        case 1: return 1
-        case _: return sum(fib.map([[n - 2, salt], [n - 1, salt]]))
+        case 0 | 1:
+            return n
+        case _:
+            return sum(fib.map([[n - 2, salt], [n - 1, salt]]))
 
 @task
 def fib_and_print(n: str, salt = None):
@@ -75,7 +73,7 @@ def cmd(f):
 @cmd
 def worker():
     init_brrr(False)
-    wrrrk(1)
+    brrr.wrrrk(1)
 
 @cmd
 def server():

dynamodb.service.nix

@@ -0,0 +1,41 @@
+# Copyright © 2024  Brrr Authors
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+# Dynamodb module for process-compose-flake
+
+{ config, pkgs, lib, ... }: {
+  options.services.dynamodb = with lib.types; {
+    enable = lib.mkEnableOption "Enable Dynamodb local service";
+    args = lib.mkOption {
+      default = [];
+      type = listOf str;
+    };
+    dataDir = lib.mkOption {
+      default = "data/dynamodb";
+      type = str;
+    };
+  };
+  config = let
+    cfg = config.services.dynamodb;
+  in
+    lib.mkIf cfg.enable {
+      settings.processes.dynamodb.command = let
+        bin = lib.getExe pkgs.dynamodb-local;
+        dir = lib.escapeShellArg cfg.dataDir;
+      in ''
+        mkdir -p ${dir}
+        ${bin} -dbPath ${dir} ${lib.escapeShellArgs cfg.args}
+      '';
+    };
+}

flake.nix

@@ -46,33 +46,41 @@
       inputs.devshell.flakeModule
     ];
     # A reusable process-compose module (for flake-parts) with either a full
+
     # demo environment, or just the dependencies if you want to run a server
     # manually.
     flake = {
-      processComposeModules.default = { pkgs, ... }: {
-        imports = [
-          ./localstack.nix
-          (inputs.services-flake.lib.multiService ./brrr-demo.nix)
-        ];
-        services = let
-          demoEnv = {
-            AWS_ENDPOINT_URL = "http://localhost:4566";
-            AWS_ACCESS_KEY_ID = "000000000000";
-            AWS_SECRET_ACCESS_KEY = "localstack-foo";
-            AWS_DEFAULT_REGION = "us-east-1";
-          };
-        in {
-          redis.r1.enable = true;
-          localstack.enable = true;
-          brrr-demo.worker = {
-            package = self.packages.${pkgs.system}.brrr-demo;
-            args = [ "worker" ];
-            environment = demoEnv;
-          };
-          brrr-demo.server = {
-            package = self.packages.${pkgs.system}.brrr-demo;
-            args = [ "server" ];
-            environment = demoEnv;
+      processComposeModules = {
+        brrr-demo = inputs.services-flake.lib.multiService ./brrr-demo.service.nix;
+        dynamodb = import ./dynamodb.service.nix;
+        localstack = import ./localstack.service.nix;
+         default = { pkgs, ... }: {
+          imports = with self.processComposeModules; [
+            brrr-demo
+            dynamodb
+            # Unused for now but will probably be reintroduced for an SQS demo
+            # soon.
+            localstack
+          ];
+          services = let
+            demoEnv = {
+              AWS_ENDPOINT_URL = "http://localhost:8000";
+              AWS_ACCESS_KEY_ID = "000000000000";
+              AWS_SECRET_ACCESS_KEY = "fake";
+            };
+          in {
+            redis.r1.enable = true;
+            dynamodb.enable = true;
+            brrr-demo.worker = {
+              package = self.packages.${pkgs.system}.brrr-demo;
+              args = [ "worker" ];
+              environment = demoEnv;
+            };
+            brrr-demo.server = {
+              package = self.packages.${pkgs.system}.brrr-demo;
+              args = [ "server" ];
+              environment = demoEnv;
+            };
           };
         };
       };
@@ -95,6 +103,11 @@
       );
     in {
       config = {
+        _module.args.pkgs = import inputs.nixpkgs {
+          inherit system;
+          # dynamodb
+          config.allowUnfree = true;
+        };
         process-compose.demo = {
           imports = [
             inputs.services-flake.processComposeModules.default
@@ -149,16 +162,30 @@
             config.Entrypoint = [ "${lib.getExe pkg}" ];
           };
         };
-        checks.pytest = pkgs.stdenvNoCC.mkDerivation {
-          name = "pytest";
-          nativeBuildInputs = [ self'.packages.dev ];
-          src = lib.cleanSource ./.;
-          buildPhase = ''
-            pytest
-          '';
-          installPhase = ''
-            touch $out
-          '';
+        checks = {
+          pytest = pkgs.stdenvNoCC.mkDerivation {
+            name = "pytest";
+            nativeBuildInputs = [ self'.packages.dev ];
+            src = lib.cleanSource ./.;
+            buildPhase = ''
+              pytest
+            '';
+            installPhase = ''
+              touch $out
+            '';
+          };
+          ruff = pkgs.stdenvNoCC.mkDerivation {
+            name = "ruff";
+            nativeBuildInputs = [ self'.packages.dev ];
+            src = lib.cleanSource ./.;
+            # Don’t check tests for now though we should
+            buildPhase = ''
+              ruff check src
+            '';
+            installPhase = ''
+              touch $out
+            '';
+          };
         };
         devshells = {
           impure = {

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "brrr"
-version = "0.0.3"
+version = "0.0.4"
 description = "Horizontally scalable workflow scheduling with pluggable backends"
 authors = [
     {name = "Hraban Luyat", email = "[email protected]"},

src/brrr/__init__.py

@@ -1,10 +1,12 @@
 from .brrr import Brrr
 
-# For ergonomics, we provide a singleton and a bunch of proxies as the module interface
-brrr = Brrr()
+# For ergonomics, we provide a singleton and a bunch of proxies as the module interface.
+_brrr = Brrr()
 
-setup = brrr.setup
-gather = brrr.gather
-wrrrk = brrr.wrrrk
-task = brrr.register_task
-schedule = brrr.schedule
+setup = _brrr.setup
+gather = _brrr.gather
+read = _brrr.read
+wrrrk = _brrr.wrrrk
+task = _brrr.register_task
+tasks = _brrr.tasks
+schedule = _brrr.schedule

src/brrr/backends/dynamo.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import os
 import typing
 
 from ..store import CompareMismatch, MemKey, Store
@@ -103,7 +102,7 @@ def compare_and_delete(self, key: MemKey, expected: bytes):
                 ExpressionAttributeNames={"#value": "value"},
                 ExpressionAttributeValues={":expected": {"B": expected}},
             )
-        except self.client.exceptions.ConditionalCheckFailedException as e:
+        except self.client.exceptions.ConditionalCheckFailedException:
             raise CompareMismatch
 
     def create_table(self):