From fb60113c2e8a7f0676af545c99f073a335c255f3 Mon Sep 17 00:00:00 2001 From: jinser Date: Mon, 1 Jul 2024 16:08:12 +0800 Subject: [PATCH] chore: support nix shell dev environment (#17462) Signed-off-by: Richard Chien Co-authored-by: Richard Chien --- .gitignore | 2 + develop/nix/devshell.nix | 60 +++++++++++++ develop/nix/flake.lock | 188 +++++++++++++++++++++++++++++++++++++++ develop/nix/flake.nix | 64 +++++++++++++ develop/nix/overlays.nix | 14 +++ docs/developer-guide.md | 25 ++++++ 6 files changed, 353 insertions(+) create mode 100644 develop/nix/devshell.nix create mode 100644 develop/nix/flake.lock create mode 100644 develop/nix/flake.nix create mode 100644 develop/nix/overlays.nix diff --git a/.gitignore b/.gitignore index 855bf21820af8..2c77ad21d9e74 100644 --- a/.gitignore +++ b/.gitignore @@ -81,3 +81,5 @@ e2e_test/iceberg/spark-*-bin* **/poetry.lock *.slt.temp + +.direnv/ diff --git a/develop/nix/devshell.nix b/develop/nix/devshell.nix new file mode 100644 index 0000000000000..13aad08c8b711 --- /dev/null +++ b/develop/nix/devshell.nix @@ -0,0 +1,60 @@ +{ inputs +, ... +}: + +{ + perSystem = { pkgs, lib, ... }: { + devshells.default = + let + rust-toolchain = with pkgs; + [ + ((rust-bin.fromRustupToolchainFile ../../rust-toolchain).override { + extensions = [ "rust-src" "rust-analyzer" ]; + }) + ]; + in + { + imports = [ + "${inputs.devshell}/extra/language/rust.nix" + ]; + language.rust.enableDefaultToolchain = false; + packages = rust-toolchain + # See the dependencies list in docs/developer-guide.md + ++ (with pkgs; [ + gcc + lld + protobuf + pkg-config + cyrus_sasl.out + + gnumake + cmake + maven + jdk17_headless + + tmux + postgresql + patchelf + ]); + env = [ + { + name = "PKG_CONFIG_PATH"; + value = lib.concatStringsSep ":" ( + map (pkg: "${pkg}/lib/pkgconfig") (with pkgs; [ + openssl.dev + cyrus_sasl.dev + ]) + ); + } + { + name = "LD_LIBRARY_PATH"; + value = lib.makeLibraryPath (with pkgs; [ + openssl + libgcc.lib + cyrus_sasl.out + ]); + } + ]; + }; + }; +} diff --git a/develop/nix/flake.lock b/develop/nix/flake.lock new file mode 100644 index 0000000000000..8ef4b42e5ed89 --- /dev/null +++ b/develop/nix/flake.lock @@ -0,0 +1,188 @@ +{ + "nodes": { + "devshell": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + }, + "locked": { + "lastModified": 1711099426, + "narHash": "sha256-HzpgM/wc3aqpnHJJ2oDqPBkNsqWbW0WfWUO8lKu8nGk=", + "owner": "numtide", + "repo": "devshell", + "rev": "2d45b54ca4a183f2fdcf4b19c895b64fbf620ee8", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "devshell", + "type": "github" + } + }, + "flake-parts": { + "inputs": { + "nixpkgs-lib": "nixpkgs-lib" + }, + "locked": { + "lastModified": 1712014858, + "narHash": "sha256-sB4SWl2lX95bExY2gMFG5HIzvva5AVMJd4Igm+GpZNw=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "9126214d0a59633752a136528f5f3b9aa8565b7d", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1701680307, + "narHash": "sha256-kAuep2h5ajznlPMD9rnQyffWG8EM/C73lejGofXvdM8=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "4022d587cbbfd70fe950c1e2083a02621806a725", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_2": { + "inputs": { + "systems": "systems_2" + }, + "locked": { + "lastModified": 1705309234, + "narHash": "sha256-uNRRNRKmJyCRC/8y1RqBkqWBLM034y4qN7EprSdmgyA=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "1ef2e671c3b0c19053962c07dbda38332dcebf26", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1704161960, + "narHash": "sha256-QGua89Pmq+FBAro8NriTuoO/wNaUtugt29/qqA8zeeM=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "63143ac2c9186be6d9da6035fa22620018c85932", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-lib": { + "locked": { + "dir": "lib", + "lastModified": 1711703276, + "narHash": "sha256-iMUFArF0WCatKK6RzfUJknjem0H9m4KgorO/p3Dopkk=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "d8fe5e6c92d0d190646fb9f1056741a229980089", + "type": "github" + }, + "original": { + "dir": "lib", + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1712439257, + "narHash": "sha256-aSpiNepFOMk9932HOax0XwNxbA38GOUVOiXfUVPOrck=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "ff0dbd94265ac470dda06a657d5fe49de93b4599", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "devshell": "devshell", + "flake-parts": "flake-parts", + "nixpkgs": "nixpkgs_2", + "rust-overlay": "rust-overlay" + } + }, + "rust-overlay": { + "inputs": { + "flake-utils": "flake-utils_2", + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1712628742, + "narHash": "sha256-FIAlt8mbPUs8jRuh6xpFtYzDsyHzmiLNPcen8HwvD00=", + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "e7354bb9e5f68b2074e272fd5f5ac3f4848860ba", + "type": "github" + }, + "original": { + "owner": "oxalica", + "repo": "rust-overlay", + "type": "github" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_2": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/develop/nix/flake.nix b/develop/nix/flake.nix new file mode 100644 index 0000000000000..e8c02d241c5eb --- /dev/null +++ b/develop/nix/flake.nix @@ -0,0 +1,64 @@ +# A nix flake that sets up a complete RisingWave development environment. +# +# You must have already installed Nix (https://nixos.org) on your system to use this. +# Nix can be installed on Linux or MacOS; NixOS is not required. Windows is not +# directly supported, but Nix can be installed inside of WSL2 or even Docker +# containers. Please refer to https://nixos.org/download for details. +# +# You must also enable support for flakes in Nix. See the following for how to +# do so permanently: https://nixos.wiki/wiki/Flakes#Enable_flakes +# +# Usage: +# +# With Nix installed, navigate to the directory containing this flake and run +# `nix develop ./develop/nix`. +# +# You should now be dropped into a new shell with all programs and dependencies +# available to you! +# +# You can exit the development shell by typing `exit`, or using Ctrl-D. +# +# If you would like this development environment to activate automatically +# upon entering this directory in your terminal, first install `direnv` +# (https://direnv.net/). Then run `echo 'use flake ./develop/nix' >> .envrc` at +# the root of the RisingWave repo. Finally, run `direnv allow` to allow the +# contents of '.envrc' to run every time you enter this directory. Voilà! +# +# note: If you don't want to see git untracked .envrc files bother you, +# you can run `echo '.envrc' >> .git/info/exclude` in the root of project +# to make it ignored locally. + +{ + description = '' + RisingWave the Cloud-native SQL stream processing, analytics, and management. + ''; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + flake-parts.url = "github:hercules-ci/flake-parts"; + + devshell.url = "github:numtide/devshell"; + + rust-overlay = { + url = "github:oxalica/rust-overlay"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + }; + + outputs = inputs@{ flake-parts, ... }: + flake-parts.lib.mkFlake { inherit inputs; } { + imports = [ + inputs.devshell.flakeModule + ./overlays.nix + ./devshell.nix + ]; + + systems = [ + "x86_64-linux" + "aarch64-linux" + "aarch64-darwin" + "x86_64-darwin" + ]; + }; +} + diff --git a/develop/nix/overlays.nix b/develop/nix/overlays.nix new file mode 100644 index 0000000000000..f37c8312cb44e --- /dev/null +++ b/develop/nix/overlays.nix @@ -0,0 +1,14 @@ +{ inputs +, ... +}: + +{ + perSystem = { pkgs, system, ... }: { + _module.args.pkgs = import inputs.nixpkgs { + inherit system; + overlays = [ + (import inputs.rust-overlay) + ]; + }; + }; +} diff --git a/docs/developer-guide.md b/docs/developer-guide.md index 49ac5c387d0ff..138565679b9f0 100644 --- a/docs/developer-guide.md +++ b/docs/developer-guide.md @@ -17,6 +17,9 @@ http://ecotrust-canada.github.io/markdown-toc/ - [Read the design docs](#read-the-design-docs) - [Learn about the code structure](#learn-about-the-code-structure) - [Set up the development environment](#set-up-the-development-environment) + * [macOS](#macos) + * [Debian-based Linux](#debian-based-linux) + * [nix shell](#nix-shell) - [Start and monitor a dev cluster](#start-and-monitor-a-dev-cluster) * [Tips for compilation](#tips-for-compilation) * [Configure additional components](#configure-additional-components) @@ -73,6 +76,8 @@ RisingWave can be built on macOS and Linux. To develop RisingWave, you need the * LLVM (For macOS only, to workaround some bugs in macOS toolchain. See https://github.com/risingwavelabs/risingwave/issues/6205) * Python (>= 3.12) (Optional, only required by `embedded-python-udf` feature) +### macOS + To install the dependencies on macOS, run: ```shell @@ -80,6 +85,8 @@ brew install postgresql cmake protobuf tmux cyrus-sasl llvm openssl@3 curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh ``` +### Debian-based Linux + To install the dependencies on Debian-based Linux systems, run: ```shell @@ -87,6 +94,24 @@ sudo apt install make build-essential cmake protobuf-compiler curl postgresql-cl curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh ``` +### nix shell + +If you use nix, you can also enter the nix shell via: + +```shell +nix develop ./develop/nix +``` + +All dependencies will be automatically downloaded and configured. + +You can also use [direnv](https://github.com/direnv/direnv) to automatically enter the nix shell: + +```shell +direnv allow +``` + +Check out [flake.nix](../develop/nix/flake.nix) to read more information! + Then you'll be able to compile and start RisingWave! > [!NOTE]