feat: Build bunq2ynab image with nix and Bazel (#544)

* Setup nix in BuildBuddy * Build Python 3.10 base image with nix * Use rules_pycross for Python dependency management * Setup Python container platform * Create bunq2ynab docker image * Upgrade Bazel * Align VSCode and lefthook prettier config
vgijssel · Oct 30, 2023 · 65fa40b · 65fa40b
1 parent 0394c25
commit 65fa40b
Show file tree

Hide file tree

Showing 28 changed files with 6,339 additions and 2,052 deletions.
diff --git a/.bazelversion b/.bazelversion
@@ -1 +1 @@
-6.3.2
+6.4.0
diff --git a/.prettierrc b/.prettierrc
@@ -0,0 +1,6 @@
+{
+  "trailingComma": "es5",
+  "tabWidth": 2,
+  "semi": true,
+  "singleQuote": false
+}
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -16,5 +16,6 @@
     "editor.defaultFormatter": "ms-azuretools.vscode-docker"
   },
   "python.formatting.provider": "none",
-  "devbox.autoShellOnTerminal": false
+  "devbox.autoShellOnTerminal": true,
+  "nix.enableLanguageServer": false
 }
diff --git a/BUILD.bazel b/BUILD.bazel
@@ -1,5 +1,12 @@
 load("@bazel_skylib//rules:common_settings.bzl", "bool_flag")
-load("@rules_task//tools:defs.bzl", "compile_pip_requirements")
+load("@rules_task//:defs.bzl", "cmd", "task")
+load(
+    "@jvolkman_rules_pycross//pycross:defs.bzl",
+    "pycross_lock_file",
+    "pycross_pdm_lock_model",
+    "pycross_target_environment",
+)
+load("@bazel_skylib//rules:diff_test.bzl", "diff_test")
 
 package(default_visibility = ["//visibility:public"])
 
@@ -17,38 +24,170 @@ config_setting(
     },
 )
 
+sh_binary(
+    name = "setup_ci",
+    srcs = [
+        "setup_ci.sh",
+    ],
+)
+
+_darwin_arm64 = [
+    "@platforms//os:macos",
+    "@platforms//cpu:arm64",
+]
+
+_linux_amd64 = [
+    "@platforms//os:linux",
+    "@platforms//cpu:x86_64",
+]
+
+_linux_arm64 = [
+    "@platforms//os:linux",
+    "@platforms//cpu:arm64",
+]
+
 config_setting(
     name = "is_linux_amd64",
-    constraint_values = [
-        "@platforms//cpu:x86_64",
-        "@platforms//os:linux",
-    ],
+    constraint_values = _linux_amd64,
 )
 
 config_setting(
     name = "is_linux_arm64",
-    constraint_values = [
-        "@platforms//cpu:aarch64",
-        "@platforms//os:linux",
-    ],
+    constraint_values = _linux_arm64,
 )
 
 config_setting(
     name = "is_darwin_arm64",
-    constraint_values = [
-        "@platforms//cpu:aarch64",
-        "@platforms//os:macos",
+    constraint_values = _darwin_arm64,
+)
+
+platform(
+    name = "darwin_arm64",
+    constraint_values = _darwin_arm64,
+)
+
+platform(
+    name = "linux_x86_64",
+    constraint_values = _linux_amd64,
+)
+
+platform(
+    name = "linux_arm64",
+    constraint_values = _linux_arm64,
+)
+
+pycross_target_environment(
+    name = "python_darwin_arm64",
+    abis = ["cp310"],
+    platforms = [
+        "macosx_12_0_arm64",
+        "macosx_13_0_arm64",
+        "macosx_14_0_arm64",
+    ],
+    python_compatible_with = _darwin_arm64,
+    version = "3.10.13",
+)
+
+pycross_target_environment(
+    name = "python_linux_amd64",
+    abis = ["cp310"],
+    platforms = [
+        "linux_x86_64",
+        "manylinux2014_x86_64",
+    ] + [
+        "manylinux_2_%s_x86_64" % str(i)
+        for i in range(17, 35)
+    ],
+    python_compatible_with = _linux_amd64,
+    version = "3.10.13",
+)
+
+pycross_target_environment(
+    name = "python_linux_arm64",
+    abis = ["cp310"],
+    platforms = [
+        "linux_aarch64",
+        "manylinux2014_aarch64",
+    ] + [
+        "manylinux_2_%s_aarch64" % str(i)
+        for i in range(17, 35)
+    ],
+    python_compatible_with = _linux_arm64,
+    version = "3.10.13",
+)
+
+pycross_pdm_lock_model(
+    name = "pdm-setup_lock_model",
+    dev = True,
+    groups = [
+        "default",
     ],
+    lock_file = "pdm.lock",
+    project_file = "pyproject.toml",
 )
 
-compile_pip_requirements(
-    name = "requirements",
-    extra_args = [
-        "--allow-unsafe",
+pycross_lock_file(
+    name = "pdm-setup_lock",
+    out = "pdm-setup_lock_gen.bzl",
+    default_alias_single_version = True,
+    lock_model_file = ":pdm-setup_lock_model",
+    package_build_dependencies = {
+        "ulid-transform": [
+            "wheel",
+            "setuptools",
+            "poetry-core",
+        ],
+        "python-slugify": [
+            "wheel",
+            "setuptools",
+        ],
+        "mock-open": [
+            "wheel",
+            "setuptools",
+        ],
+        "lru-dict": [
+            "wheel",
+            "setuptools",
+            "poetry-core",
+        ],
+        "pytest-test-groups": [
+            "wheel",
+            "setuptools",
+        ],
+        "colored": [
+            "wheel",
+            "setuptools",
+        ],
+        "paho-mqtt": [
+            "wheel",
+            "setuptools",
+        ],
+        "home-assistant-bluetooth": [
+            "wheel",
+            "setuptools",
+            "poetry-core",
+        ],
+    },
+    pypi_index = "https://pypi.org",
+    target_environments = [
+        ":python_darwin_arm64",
+        ":python_linux_amd64",
+        ":python_linux_arm64",
     ],
-    hidden_args = [
-        "--verbose",
+)
+
+task(
+    name = "pdm-setup.update",
+    cmds = [
+        "cp -fv $LOCK_FILE $BUILD_WORKSPACE_DIRECTORY/pdm-setup_lock.bzl",
     ],
-    requirements_in = "requirements.in",
-    requirements_txt = "requirements.txt",
+    env = {
+        "LOCK_FILE": cmd.file(":pdm-setup_lock"),
+    },
+)
+
+diff_test(
+    name = "pdm-setup_test",
+    file1 = "pdm-setup_lock.bzl",
+    file2 = ":pdm-setup_lock",
 )
diff --git a/MODULE.bazel b/MODULE.bazel
@@ -8,11 +8,18 @@ module(
     version = "1.0",
 )
 
+# It's important that this toolchain is registered before the toolchains from rules_python
+# so this one can be picked when we run Python inside a container.
+register_toolchains(
+    "//tools/python:python_container_py_toolchain",
+)
+
 # ------------------------------------ platforms ------------------------------------ #
 bazel_dep(name = "platforms", version = "0.0.8")
 
 # ------------------------------------ rules_skylib ------------------------------------ #
 bazel_dep(name = "bazel_skylib", version = "1.4.2")
+bazel_dep(name = "aspect_bazel_lib", version = "2.0.0-rc1")
 
 # ------------------------------------ rules_pkg ------------------------------------ #
 bazel_dep(name = "rules_pkg", version = "0.9.1")
@@ -38,12 +45,5 @@ python.toolchain(
 
 use_repo(python, "python_versions")
 
-pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
-
-pip.parse(
-    hub_name = "pip-setup",
-    python_version = "3.10",
-    requirements_lock = "//:requirements.txt",
-)
-
-use_repo(pip, "pip-setup")
+# ------------------------------------ rules_oci ------------------------------------ #
+bazel_dep(name = "rules_oci", version = "1.4.0")
diff --git a/WORKSPACE.bzlmod b/WORKSPACE.bzlmod
@@ -45,6 +45,39 @@ container_pull(
     repository = "library/python:3.10.8-bullseye",
 )
 
+# ------------------------------------ rules_pycross ------------------------------------ #
+http_archive(
+    name = "jvolkman_rules_pycross",
+    sha256 = "5b51cfa4890d6de29b6a0976248e8139f69354bf29924308093a039d93ab630f",
+    strip_prefix = "rules_pycross-0.2",
+    url = "https://github.com/jvolkman/rules_pycross/archive/refs/tags/v0.2.tar.gz",
+)
+
+load("@jvolkman_rules_pycross//pycross:repositories.bzl", "rules_pycross_dependencies")
+load("@rules_python~0.25.0~python~pythons_hub//:interpreters.bzl", "INTERPRETER_LABELS")
+
+# This is necessary to get the hermetic interpreter which is defined in MODULE.bazel
+# This would be simpler if rules_pycross would also use bzlmod.
+interpreter_internal_target = INTERPRETER_LABELS["python_3_10"]
+
+interpreter_target = "@rules_python~0.25.0~python~{}//:{}".format(
+    interpreter_internal_target.workspace_name,
+    interpreter_internal_target.name,
+)
+
+rules_pycross_dependencies(python_interpreter_target = interpreter_target)
+
+load("@jvolkman_rules_pycross//pycross:defs.bzl", "pycross_lock_repo")
+
+pycross_lock_repo(
+    name = "pdm-setup",
+    lock_file = "//:pdm-setup_lock.bzl",
+)
+
+load("@pdm-setup//:requirements.bzl", "install_deps")
+
+install_deps()
+
 # ------------------------------------ workstation ------------------------------------ #
 
 # https://hub.docker.com/layers/library/ubuntu/jammy-20230425/images/sha256-ca5534a51dd04bbcebe9b23ba05f389466cf0c190f1f8f182d7eea92a9671d00?context=explore
@@ -262,3 +295,34 @@ git_repository(
 #     build_file = "//tools/bunq2ynab:BUILD.repositories.bazel.tpl",
 #     path = "/workspaces/bunq2ynab",
 # )
+
+# ------------------------------------ rules_nixpkgs ------------------------------------ #
+
+http_archive(
+    name = "io_tweag_rules_nixpkgs",
+    sha256 = "b01f170580f646ee3cde1ea4c117d00e561afaf3c59eda604cf09194a824ff10",
+    strip_prefix = "rules_nixpkgs-0.9.0",
+    urls = ["https://github.com/tweag/rules_nixpkgs/archive/refs/tags/v0.9.0.tar.gz"],
+)
+
+load("@io_tweag_rules_nixpkgs//nixpkgs:repositories.bzl", "rules_nixpkgs_dependencies")
+
+rules_nixpkgs_dependencies()
+
+load("@io_tweag_rules_nixpkgs//nixpkgs:nixpkgs.bzl", "nixpkgs_git_repository", "nixpkgs_package")
+
+nixpkgs_git_repository(
+    name = "nixpkgs",
+    revision = "23.05",  # Any tag or commit hash
+    sha256 = "",  # optional sha to verify package integrity!
+)
+
+nixpkgs_package(
+    name = "python310_base_image",
+    build_file_content = """
+package(default_visibility = [ "//visibility:public" ])
+exports_files(["image.tar.gz"])
+    """,
+    nix_file = "//:python310_base_image.nix",
+    repository = "@nixpkgs//:default.nix",
+)
diff --git a/buildbuddy.yaml b/buildbuddy.yaml
@@ -10,4 +10,5 @@ actions:
         branches:
           - "*"
     bazel_commands:
+      - bazel run //:setup_ci
       - bazel test --keep_going //... @rules_task//... --config buildbuddy --config buildbuddy_rbe
diff --git a/devbox.json b/devbox.json
@@ -5,10 +5,13 @@
     "[email protected]",
     "[email protected]",
     "[email protected]",
-    "[email protected]"
+    "[email protected]",
+    "[email protected]",
+    "libarchive@latest",
+    "[email protected]",
+    "[email protected]"
   ],
   "shell": {
-    "init_hook": ["alias bazel='bazelisk'"],
-    "scripts": {}
+    "init_hook": ["alias bazel='bazelisk'", "export PATH=$HOME/bin:$PATH"]
   }
 }