From b9259ce5d1101a29c9555c9c2267353f3f203589 Mon Sep 17 00:00:00 2001 From: Hraban Luyat Date: Sat, 14 Dec 2024 14:53:33 -0500 Subject: [PATCH] release v0.0.4: queue topics and module API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit chore: set version to 0.0.4.dev chore: convert CRLF to LF in .gitignore refactor: localstack -> dynamodb-local for demo refactor: move tests out of src/ test: fix some linter bugs refactor: hide the ‘brrr’ singleton It leaks API and the name clash with the module is extremely confusing. chore: remove unused asyncrrr.py test: run linter --- .gitignore | 17 +++-- README.md | 9 +-- brrr-demo.nix => brrr-demo.service.nix | 4 +- brrr_demo.py | 16 ++-- dynamodb.service.nix | 41 ++++++++++ flake.nix | 95 +++++++++++++++--------- localstack.nix => localstack.service.nix | 0 pyproject.toml | 2 +- src/brrr/__init__.py | 16 ++-- src/brrr/asyncrrr.py | 9 --- src/brrr/backends/dynamo.py | 3 +- {src/tests => tests}/__init__.py | 0 {src/tests => tests}/test_queue.py | 0 {src/tests => tests}/test_store.py | 0 14 files changed, 135 insertions(+), 77 deletions(-) rename brrr-demo.nix => brrr-demo.service.nix (90%) create mode 100644 dynamodb.service.nix rename localstack.nix => localstack.service.nix (100%) delete mode 100644 src/brrr/asyncrrr.py rename {src/tests => tests}/__init__.py (100%) rename {src/tests => tests}/test_queue.py (100%) rename {src/tests => tests}/test_store.py (100%) diff --git a/.gitignore b/.gitignore index 126b2e4..d51687f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,10 @@ -.venv -.envrc -.terraform* -data -__pycache__ -result -.direnv +.venv +.envrc +.terraform* +data +__pycache__ +result +.direnv + +# Noise from dynamodb-local program in the working directory +dynamodb-local-metadata.json diff --git a/README.md b/README.md index c39e929..1197c00 100644 --- a/README.md +++ b/README.md @@ -56,14 +56,9 @@ Drawback: the call graph must be idempotent, meaning: for the same inputs, a tas ## Demo -### Requirements +Requires [Nix](https://nixos.org), with flakes enabled. -- Docker -- [Nix](https://nixos.org) - -### Instructions - -Make sure you have Docker running, and start the full demo: +You can start the full demo without installation: ``` $ nix run github:nobssoftware/brrr#demo diff --git a/brrr-demo.nix b/brrr-demo.service.nix similarity index 90% rename from brrr-demo.nix rename to brrr-demo.service.nix index 898fdab..792894a 100644 --- a/brrr-demo.nix +++ b/brrr-demo.service.nix @@ -12,7 +12,9 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -# Brrr-demo module for process-compose-flake +# Brrr-demo module for services-flake. Awkwardly named file because +# services-flake insists on auto-deriving the module name from the filename. +# Ok. { config, pkgs, name, lib, ... }: { options = with lib.types; { diff --git a/brrr_demo.py b/brrr_demo.py index 7441b04..8c025f9 100755 --- a/brrr_demo.py +++ b/brrr_demo.py @@ -11,7 +11,7 @@ from brrr.backends import redis as redis_, dynamo import brrr -from brrr import task, wrrrk, setup, brrr +from brrr import task @bottle.route("/") def get_or_schedule_task(task_name: str): @@ -33,9 +33,6 @@ def get_or_schedule_task(task_name: str): return {"status": "accepted"} def init_brrr(reset_backends): - # Check credentials - boto3.client('sts').get_caller_identity() - redis_client = redis.Redis(decode_responses=True) queue = redis_.RedisStream(redis_client, os.environ.get("REDIS_QUEUE_KEY", "r1")) if reset_backends: @@ -46,14 +43,15 @@ def init_brrr(reset_backends): if reset_backends: store.create_table() - setup(queue, store) + brrr.setup(queue, store) @task def fib(n: int, salt=None): match n: - case 0: return 0 - case 1: return 1 - case _: return sum(fib.map([[n - 2, salt], [n - 1, salt]])) + case 0 | 1: + return n + case _: + return sum(fib.map([[n - 2, salt], [n - 1, salt]])) @task def fib_and_print(n: str, salt = None): @@ -75,7 +73,7 @@ def cmd(f): @cmd def worker(): init_brrr(False) - wrrrk(1) + brrr.wrrrk(1) @cmd def server(): diff --git a/dynamodb.service.nix b/dynamodb.service.nix new file mode 100644 index 0000000..718824e --- /dev/null +++ b/dynamodb.service.nix @@ -0,0 +1,41 @@ +# Copyright © 2024 Brrr Authors +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +# Dynamodb module for process-compose-flake + +{ config, pkgs, lib, ... }: { + options.services.dynamodb = with lib.types; { + enable = lib.mkEnableOption "Enable Dynamodb local service"; + args = lib.mkOption { + default = []; + type = listOf str; + }; + dataDir = lib.mkOption { + default = "data/dynamodb"; + type = str; + }; + }; + config = let + cfg = config.services.dynamodb; + in + lib.mkIf cfg.enable { + settings.processes.dynamodb.command = let + bin = lib.getExe pkgs.dynamodb-local; + dir = lib.escapeShellArg cfg.dataDir; + in '' + mkdir -p ${dir} + ${bin} -dbPath ${dir} ${lib.escapeShellArgs cfg.args} + ''; + }; +} diff --git a/flake.nix b/flake.nix index 85adcb7..b78086a 100644 --- a/flake.nix +++ b/flake.nix @@ -46,33 +46,41 @@ inputs.devshell.flakeModule ]; # A reusable process-compose module (for flake-parts) with either a full + # demo environment, or just the dependencies if you want to run a server # manually. flake = { - processComposeModules.default = { pkgs, ... }: { - imports = [ - ./localstack.nix - (inputs.services-flake.lib.multiService ./brrr-demo.nix) - ]; - services = let - demoEnv = { - AWS_ENDPOINT_URL = "http://localhost:4566"; - AWS_ACCESS_KEY_ID = "000000000000"; - AWS_SECRET_ACCESS_KEY = "localstack-foo"; - AWS_DEFAULT_REGION = "us-east-1"; - }; - in { - redis.r1.enable = true; - localstack.enable = true; - brrr-demo.worker = { - package = self.packages.${pkgs.system}.brrr-demo; - args = [ "worker" ]; - environment = demoEnv; - }; - brrr-demo.server = { - package = self.packages.${pkgs.system}.brrr-demo; - args = [ "server" ]; - environment = demoEnv; + processComposeModules = { + brrr-demo = inputs.services-flake.lib.multiService ./brrr-demo.service.nix; + dynamodb = import ./dynamodb.service.nix; + localstack = import ./localstack.service.nix; + default = { pkgs, ... }: { + imports = with self.processComposeModules; [ + brrr-demo + dynamodb + # Unused for now but will probably be reintroduced for an SQS demo + # soon. + localstack + ]; + services = let + demoEnv = { + AWS_ENDPOINT_URL = "http://localhost:8000"; + AWS_ACCESS_KEY_ID = "000000000000"; + AWS_SECRET_ACCESS_KEY = "fake"; + }; + in { + redis.r1.enable = true; + dynamodb.enable = true; + brrr-demo.worker = { + package = self.packages.${pkgs.system}.brrr-demo; + args = [ "worker" ]; + environment = demoEnv; + }; + brrr-demo.server = { + package = self.packages.${pkgs.system}.brrr-demo; + args = [ "server" ]; + environment = demoEnv; + }; }; }; }; @@ -95,6 +103,11 @@ ); in { config = { + _module.args.pkgs = import inputs.nixpkgs { + inherit system; + # dynamodb + config.allowUnfree = true; + }; process-compose.demo = { imports = [ inputs.services-flake.processComposeModules.default @@ -149,16 +162,30 @@ config.Entrypoint = [ "${lib.getExe pkg}" ]; }; }; - checks.pytest = pkgs.stdenvNoCC.mkDerivation { - name = "pytest"; - nativeBuildInputs = [ self'.packages.dev ]; - src = lib.cleanSource ./.; - buildPhase = '' - pytest - ''; - installPhase = '' - touch $out - ''; + checks = { + pytest = pkgs.stdenvNoCC.mkDerivation { + name = "pytest"; + nativeBuildInputs = [ self'.packages.dev ]; + src = lib.cleanSource ./.; + buildPhase = '' + pytest + ''; + installPhase = '' + touch $out + ''; + }; + ruff = pkgs.stdenvNoCC.mkDerivation { + name = "ruff"; + nativeBuildInputs = [ self'.packages.dev ]; + src = lib.cleanSource ./.; + # Don’t check tests for now though we should + buildPhase = '' + ruff check src + ''; + installPhase = '' + touch $out + ''; + }; }; devshells = { impure = { diff --git a/localstack.nix b/localstack.service.nix similarity index 100% rename from localstack.nix rename to localstack.service.nix diff --git a/pyproject.toml b/pyproject.toml index 6196631..f2c806c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "brrr" -version = "0.0.3" +version = "0.0.4" description = "Horizontally scalable workflow scheduling with pluggable backends" authors = [ {name = "Hraban Luyat", email = "hraban@0brg.net"}, diff --git a/src/brrr/__init__.py b/src/brrr/__init__.py index b0f1e2f..9ca3265 100644 --- a/src/brrr/__init__.py +++ b/src/brrr/__init__.py @@ -1,10 +1,12 @@ from .brrr import Brrr -# For ergonomics, we provide a singleton and a bunch of proxies as the module interface -brrr = Brrr() +# For ergonomics, we provide a singleton and a bunch of proxies as the module interface. +_brrr = Brrr() -setup = brrr.setup -gather = brrr.gather -wrrrk = brrr.wrrrk -task = brrr.register_task -schedule = brrr.schedule +setup = _brrr.setup +gather = _brrr.gather +read = _brrr.read +wrrrk = _brrr.wrrrk +task = _brrr.register_task +tasks = _brrr.tasks +schedule = _brrr.schedule diff --git a/src/brrr/asyncrrr.py b/src/brrr/asyncrrr.py deleted file mode 100644 index 082a6ea..0000000 --- a/src/brrr/asyncrrr.py +++ /dev/null @@ -1,9 +0,0 @@ -import asyncio - -async def async_wrrrk(workers: int = 1): - """ - Spin up a number of worker threads - """ - return asyncio.gather( - *[asyncio.create_task(asyncio_worker()) for _ in range(workers)] - ) diff --git a/src/brrr/backends/dynamo.py b/src/brrr/backends/dynamo.py index 3a5132e..e5901ed 100644 --- a/src/brrr/backends/dynamo.py +++ b/src/brrr/backends/dynamo.py @@ -1,6 +1,5 @@ from __future__ import annotations -import os import typing from ..store import CompareMismatch, MemKey, Store @@ -103,7 +102,7 @@ def compare_and_delete(self, key: MemKey, expected: bytes): ExpressionAttributeNames={"#value": "value"}, ExpressionAttributeValues={":expected": {"B": expected}}, ) - except self.client.exceptions.ConditionalCheckFailedException as e: + except self.client.exceptions.ConditionalCheckFailedException: raise CompareMismatch def create_table(self): diff --git a/src/tests/__init__.py b/tests/__init__.py similarity index 100% rename from src/tests/__init__.py rename to tests/__init__.py diff --git a/src/tests/test_queue.py b/tests/test_queue.py similarity index 100% rename from src/tests/test_queue.py rename to tests/test_queue.py diff --git a/src/tests/test_store.py b/tests/test_store.py similarity index 100% rename from src/tests/test_store.py rename to tests/test_store.py