From 8e01e28ea463babc88a897e0d76160075dcfd669 Mon Sep 17 00:00:00 2001 From: Joep Meindertsma Date: Wed, 8 Feb 2023 16:53:37 +0100 Subject: [PATCH 1/8] - Replace `acme_lib` with `instant-acme`, drop OpenSSL dependency #192 --- CHANGELOG.md | 1 + CONTRIBUTE.md | 11 + Cargo.lock | 468 +++++++++++++-------------------------- server/Cargo.toml | 11 +- server/src/https_init.rs | 138 +++++++++++- 5 files changed, 313 insertions(+), 316 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8bb26bd02..cd87976e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ See [STATUS.md](server/STATUS.md) to learn more about which features will remain - Meta tags server side #577 - Include JSON-AD in initial response, speed up first render #511 - Remove feature to index external RDF files and search them #579 +- Replace `acme_lib` with `instant-acme`, drop OpenSSL dependency #192 ## [v0.34.0] - 2022-10-31 diff --git a/CONTRIBUTE.md b/CONTRIBUTE.md index 7c18fc064..b45401d74 100644 --- a/CONTRIBUTE.md +++ b/CONTRIBUTE.md @@ -14,6 +14,7 @@ Check out the [Roadmap](https://docs.atomicdata.dev/roadmap.html) if you want to - [Table of contents](#table-of-contents) - [Running locally](#running-locally) +- [Cross compilation](#cross-compilation) - [IDE setup (VSCode)](#ide-setup-vscode) - [Testing](#testing) - [Code coverage](#code-coverage) @@ -45,6 +46,16 @@ Since `atomic-server` is developed in conjunction with the typescript / react `a - Visit your `localhost` in your locally running `atomic-data-browser` instance: (e.g. `http://localhost:8080/app/show?subject=http%3A%2F%2Flocalhost`) - use `cargo watch -- cargo run` to automatically recompile `atomic-server` when you push new assets using `pmpm build-server` in `atomic-data-browser`. This can be useful if you're debugging specific features that you can't reproduce while the front-end is hosted in vite. +## Cross compilation + +If you want to build `atomic-server` for some other target (e.g. building for linux from macOS), you can use the `cross` crate. + +```sh +cargo install cross +# make sure docker is running! +cross build --target x86_64-unknown-linux-musl +``` + ## IDE setup (VSCode) This project is primarily being developed in VSCode. diff --git a/Cargo.lock b/Cargo.lock index a486f2151..173df1df1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,22 +2,6 @@ # It is not intended for manual editing. version = 3 -[[package]] -name = "acme-lib" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "292ac9d513052341a7f5bdae61f31c4dc93c1dce2598508f52709df08cecc8b0" -dependencies = [ - "base64", - "lazy_static", - "log", - "openssl", - "serde", - "serde_json", - "time 0.1.44", - "ureq 1.5.5", -] - [[package]] name = "actix" version = "0.13.0" @@ -109,7 +93,7 @@ dependencies = [ "actix-tls", "actix-utils", "ahash", - "base64", + "base64 0.13.0", "bitflags", "brotli", "bytes", @@ -129,7 +113,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "rand 0.8.5", - "sha1 0.10.4", + "sha1", "smallvec", "tracing", "zstd", @@ -232,7 +216,7 @@ dependencies = [ "pin-project-lite", "tokio-rustls", "tokio-util", - "webpki-roots 0.22.4", + "webpki-roots", ] [[package]] @@ -265,7 +249,7 @@ dependencies = [ "bytes", "bytestring", "cfg-if", - "cookie 0.16.0", + "cookie", "derive_more", "encoding_rs", "futures-core", @@ -481,7 +465,6 @@ dependencies = [ name = "atomic-server" version = "0.34.0" dependencies = [ - "acme-lib", "actix", "actix-cors", "actix-files", @@ -492,7 +475,7 @@ dependencies = [ "actix-web-static-files", "assert_cmd", "atomic_lib", - "base64", + "base64 0.13.0", "chrono", "clap 3.2.20", "colored", @@ -500,14 +483,16 @@ dependencies = [ "directories", "dotenv", "futures", + "instant-acme", "opentelemetry", "opentelemetry-jaeger", "percent-encoding", "promptly", + "rcgen", "regex", "rio_api", "rio_turtle", - "rustls 0.20.6", + "rustls", "rustls-pemfile", "sanitize-filename", "serde", @@ -522,7 +507,7 @@ dependencies = [ "tracing-log", "tracing-opentelemetry", "tracing-subscriber", - "ureq 2.5.0", + "ureq", "urlencoding", ] @@ -542,7 +527,7 @@ dependencies = [ name = "atomic_lib" version = "0.34.0" dependencies = [ - "base64", + "base64 0.13.0", "bincode", "criterion", "directories", @@ -562,7 +547,7 @@ dependencies = [ "sled", "toml", "tracing", - "ureq 2.5.0", + "ureq", "url", "urlencoding", ] @@ -603,16 +588,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] -name = "base-x" -version = "0.2.11" +name = "base64" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" [[package]] name = "base64" -version = "0.13.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" +checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" [[package]] name = "bincode" @@ -979,29 +964,12 @@ dependencies = [ "winapi", ] -[[package]] -name = "const_fn" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbdcdcb6d86f71c5e97409ad45898af11cbc995b4ee8112d59095a28d376c935" - [[package]] name = "convert_case" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" -[[package]] -name = "cookie" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03a5d7b21829bc7b4bf4754a978a241ae54ea55a40f92bb20216e54096f4b951" -dependencies = [ - "percent-encoding", - "time 0.2.27", - "version_check", -] - [[package]] name = "cookie" version = "0.16.0" @@ -1013,22 +981,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "cookie_store" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3818dfca4b0cb5211a659bbcbb94225b7127407b2b135e650d717bfb78ab10d3" -dependencies = [ - "cookie 0.14.4", - "idna 0.2.3", - "log", - "publicsuffix", - "serde", - "serde_json", - "time 0.2.27", - "url", -] - [[package]] name = "core-foundation" version = "0.9.3" @@ -1439,12 +1391,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "discard" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" - [[package]] name = "dispatch" version = "0.2.0" @@ -2199,6 +2145,17 @@ dependencies = [ "itoa 1.0.3", ] +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + [[package]] name = "http-range" version = "0.1.5" @@ -2217,6 +2174,44 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" +[[package]] +name = "hyper" +version = "0.14.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e011372fa0b68db8350aa7a248930ecc7839bf46d8485577d69f117a75f164c" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa 1.0.3", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c" +dependencies = [ + "http", + "hyper", + "rustls", + "rustls-native-certs", + "tokio", + "tokio-rustls", +] + [[package]] name = "iai" version = "0.1.1" @@ -2253,17 +2248,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" -[[package]] -name = "idna" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" -dependencies = [ - "matches", - "unicode-bidi", - "unicode-normalization", -] - [[package]] name = "idna" version = "0.3.0" @@ -2345,6 +2329,21 @@ dependencies = [ "web-sys", ] +[[package]] +name = "instant-acme" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b610f3da5efb8193805de5f7b74110ca0eb76ae550a03e0a11a24d29e3230ed" +dependencies = [ + "base64 0.21.0", + "hyper", + "hyper-rustls", + "ring", + "serde", + "serde_json", + "thiserror", +] + [[package]] name = "integer-encoding" version = "3.0.4" @@ -3392,6 +3391,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd" +[[package]] +name = "pem" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8835c273a76a90455d7344889b0964598e3316e2a79ede8e36f16bdcf2228b8" +dependencies = [ + "base64 0.13.0", +] + [[package]] name = "percent-encoding" version = "2.2.0" @@ -3550,7 +3558,7 @@ version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd39bc6cdc9355ad1dc5eeedefee696bb35c34caf21768741e81826c0bbd7225" dependencies = [ - "base64", + "base64 0.13.0", "indexmap", "line-wrap", "serde", @@ -3720,25 +3728,6 @@ dependencies = [ "rustyline", ] -[[package]] -name = "publicsuffix" -version = "1.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95b4ce31ff0a27d93c8de1849cf58162283752f065a90d508f1105fa6c9a213f" -dependencies = [ - "idna 0.2.3", - "url", -] - -[[package]] -name = "qstring" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d464fae65fff2680baf48019211ce37aaec0c78e9264c84a3e484717f965104e" -dependencies = [ - "percent-encoding", -] - [[package]] name = "quote" version = "1.0.21" @@ -3872,6 +3861,18 @@ dependencies = [ "num_cpus", ] +[[package]] +name = "rcgen" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffbe84efe2f38dea12e9bfc1f65377fdf03e53a18cb3b995faedf7934c7e785b" +dependencies = [ + "pem", + "ring", + "time 0.3.14", + "yasna", +] + [[package]] name = "redox_syscall" version = "0.2.16" @@ -4000,15 +4001,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "rustc_version" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" -dependencies = [ - "semver 0.9.0", -] - [[package]] name = "rustc_version" version = "0.3.3" @@ -4043,27 +4035,26 @@ dependencies = [ [[package]] name = "rustls" -version = "0.19.1" +version = "0.20.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7" +checksum = "5aab8ee6c7097ed6057f43c187a62418d0c05a4bd5f18b3571db50ee0f9ce033" dependencies = [ - "base64", "log", "ring", - "sct 0.6.1", - "webpki 0.21.4", + "sct", + "webpki", ] [[package]] -name = "rustls" -version = "0.20.6" +name = "rustls-native-certs" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aab8ee6c7097ed6057f43c187a62418d0c05a4bd5f18b3571db50ee0f9ce033" +checksum = "0167bac7a9f490495f3c33013e7722b53cb087ecbe082fb0c6387c96f634ea50" dependencies = [ - "log", - "ring", - "sct 0.7.0", - "webpki 0.22.0", + "openssl-probe", + "rustls-pemfile", + "schannel", + "security-framework", ] [[package]] @@ -4072,7 +4063,7 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0864aeff53f8c05aa08d86e5ef839d3dfcf07aeba2db32f12db0ef716e87bd55" dependencies = [ - "base64", + "base64 0.13.0", ] [[package]] @@ -4158,16 +4149,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" -[[package]] -name = "sct" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "sct" version = "0.7.0" @@ -4221,22 +4202,13 @@ dependencies = [ "thin-slice", ] -[[package]] -name = "semver" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" -dependencies = [ - "semver-parser 0.7.0", -] - [[package]] name = "semver" version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f301af10236f6df4160f7c3f04eec6dbc70ace82d23326abad5edee88801c6b6" dependencies = [ - "semver-parser 0.10.2", + "semver-parser", ] [[package]] @@ -4248,12 +4220,6 @@ dependencies = [ "serde", ] -[[package]] -name = "semver-parser" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" - [[package]] name = "semver-parser" version = "0.10.2" @@ -4381,15 +4347,6 @@ dependencies = [ "stable_deref_trait", ] -[[package]] -name = "sha1" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1da05c97445caa12d05e848c4a4fcbbea29e748ac28f7e80e9b010392063770" -dependencies = [ - "sha1_smol", -] - [[package]] name = "sha1" version = "0.10.4" @@ -4401,12 +4358,6 @@ dependencies = [ "digest", ] -[[package]] -name = "sha1_smol" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" - [[package]] name = "sha2" version = "0.10.5" @@ -4533,15 +4484,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" -[[package]] -name = "standback" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e113fb6f3de07a243d434a56ec6f186dfd51cb08448239fe7bcae73f87ff28ff" -dependencies = [ - "version_check", -] - [[package]] name = "state" version = "0.5.3" @@ -4562,55 +4504,6 @@ dependencies = [ "path-slash", ] -[[package]] -name = "stdweb" -version = "0.4.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5" -dependencies = [ - "discard", - "rustc_version 0.2.3", - "stdweb-derive", - "stdweb-internal-macros", - "stdweb-internal-runtime", - "wasm-bindgen", -] - -[[package]] -name = "stdweb-derive" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" -dependencies = [ - "proc-macro2", - "quote", - "serde", - "serde_derive", - "syn", -] - -[[package]] -name = "stdweb-internal-macros" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11" -dependencies = [ - "base-x", - "proc-macro2", - "quote", - "serde", - "serde_derive", - "serde_json", - "sha1 0.6.1", - "syn", -] - -[[package]] -name = "stdweb-internal-runtime" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0" - [[package]] name = "str-buf" version = "1.0.6" @@ -4729,7 +4622,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d3da9d47a874779d5a97c5f01c2d0a85fc96e38ed6d51ad21a3cf6d6eb8c47" dependencies = [ "async-trait", - "base64", + "base64 0.13.0", "bitpacking", "byteorder", "census", @@ -4878,7 +4771,7 @@ checksum = "e1a56a8b125069c2682bd31610109b4436c050c74447bee1078217a0325c1add" dependencies = [ "anyhow", "attohttpc", - "base64", + "base64 0.13.0", "cocoa", "dirs-next", "embed_plist", @@ -4949,7 +4842,7 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16d62a3c8790d6cba686cea6e3f7f569d12c662c3274c2d165a4fd33e3871b72" dependencies = [ - "base64", + "base64 0.13.0", "brotli", "ico", "json-patch", @@ -5183,21 +5076,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "time" -version = "0.2.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4752a97f8eebd6854ff91f1c1824cd6160626ac4bd44287f7f4ea2035a02a242" -dependencies = [ - "const_fn", - "libc", - "standback", - "stdweb", - "time-macros 0.1.1", - "version_check", - "winapi", -] - [[package]] name = "time" version = "0.3.14" @@ -5208,17 +5086,7 @@ dependencies = [ "libc", "num_threads", "serde", - "time-macros 0.2.4", -] - -[[package]] -name = "time-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "957e9c6e26f12cb6d0dd7fc776bb67a706312e7299aed74c8dd5b17ebb27e2f1" -dependencies = [ - "proc-macro-hack", - "time-macros-impl", + "time-macros", ] [[package]] @@ -5227,19 +5095,6 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42657b1a6f4d817cda8e7a0ace261fe0cc946cf3a80314390b22cc61ae080792" -[[package]] -name = "time-macros-impl" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd3c141a1b43194f3f56a1411225df8646c55781d5f26db825b3d98507eb482f" -dependencies = [ - "proc-macro-hack", - "proc-macro2", - "quote", - "standback", - "syn", -] - [[package]] name = "tinytemplate" version = "1.2.1" @@ -5291,9 +5146,9 @@ version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" dependencies = [ - "rustls 0.20.6", + "rustls", "tokio", - "webpki 0.22.0", + "webpki", ] [[package]] @@ -5319,6 +5174,12 @@ dependencies = [ "serde", ] +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + [[package]] name = "tracing" version = "0.1.36" @@ -5440,6 +5301,12 @@ dependencies = [ "serde_json", ] +[[package]] +name = "try-lock" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" + [[package]] name = "twoway" version = "0.2.2" @@ -5516,40 +5383,21 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" -[[package]] -name = "ureq" -version = "1.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b8b063c2d59218ae09f22b53c42eaad0d53516457905f5235ca4bc9e99daa71" -dependencies = [ - "base64", - "chunked_transfer", - "cookie 0.14.4", - "cookie_store", - "log", - "once_cell", - "qstring", - "rustls 0.19.1", - "url", - "webpki 0.21.4", - "webpki-roots 0.21.1", -] - [[package]] name = "ureq" version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b97acb4c28a254fd7a4aeec976c46a7fa404eac4d7c134b30c75144846d7cb8f" dependencies = [ - "base64", + "base64 0.13.0", "chunked_transfer", "flate2", "log", "once_cell", - "rustls 0.20.6", + "rustls", "url", - "webpki 0.22.0", - "webpki-roots 0.22.4", + "webpki", + "webpki-roots", ] [[package]] @@ -5559,7 +5407,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" dependencies = [ "form_urlencoded", - "idna 0.3.0", + "idna", "percent-encoding", "serde", ] @@ -5680,6 +5528,16 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + [[package]] name = "wasi" version = "0.9.0+wasi-snapshot-preview1" @@ -5821,16 +5679,6 @@ dependencies = [ "system-deps 6.0.2", ] -[[package]] -name = "webpki" -version = "0.21.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "webpki" version = "0.22.0" @@ -5841,22 +5689,13 @@ dependencies = [ "untrusted", ] -[[package]] -name = "webpki-roots" -version = "0.21.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aabe153544e473b775453675851ecc86863d2a81d786d741f6b76778f2a48940" -dependencies = [ - "webpki 0.21.4", -] - [[package]] name = "webpki-roots" version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1c760f0d366a6c24a02ed7816e23e691f5d92291f94d15e836006fd11b04daf" dependencies = [ - "webpki 0.22.0", + "webpki", ] [[package]] @@ -6257,6 +6096,15 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" +[[package]] +name = "yasna" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aed2e7a52e3744ab4d0c05c20aa065258e84c49fd4226f5191b2ed29712710b4" +dependencies = [ + "time 0.3.14", +] + [[package]] name = "zeroize" version = "1.5.7" diff --git a/server/Cargo.toml b/server/Cargo.toml index 79900704b..2eb165846 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -44,9 +44,13 @@ tracing-log = "0.1" ureq = "2" urlencoding = "2" -[dependencies.acme-lib] +[dependencies.instant-acme] optional = true -version = "0.8" +version = "0.1" + +[dependencies.rcgen] +optional = true +version = "0.10" [dependencies.tracing-opentelemetry] optional = true @@ -105,8 +109,7 @@ assert_cmd = "2" [features] default = ["https", "telemetry"] -https = ["rustls"] -https_init = ["acme-lib"] +https = ["rustls", "instant-acme", "rcgen"] process-management = ["sysinfo"] telemetry = ["tracing-opentelemetry", "opentelemetry", "opentelemetry-jaeger"] diff --git a/server/src/https_init.rs b/server/src/https_init.rs index 3cdcccd05..07fa92227 100644 --- a/server/src/https_init.rs +++ b/server/src/https_init.rs @@ -5,6 +5,8 @@ use acme_lib::create_p384_key; use acme_lib::persist::FilePersist; use acme_lib::{Directory, DirectoryUrl, Error}; use actix_web::{App, HttpServer}; +use instant_acme::OrderStatus; +use tracing::{error, info}; use std::sync::mpsc; use std::{ @@ -66,14 +68,16 @@ pub async fn cert_init_server(config: &crate::config::Config) -> AtomicServerRes tracing::info!("Server for HTTP initialization running correctly"); } - request_cert(config).map_err(|e| format!("Certification init failed: {}", e))?; + request_cert(config) + .await + .map_err(|e| format!("Certification init failed: {}", e))?; tracing::warn!("HTTPS TLS Cert init sucesful! Stopping HTTP server, starting HTTPS..."); handle.stop(true).await; Ok(()) } /// Writes keys to disk using LetsEncrypt -fn request_cert(config: &crate::config::Config) -> Result<(), Error> { +fn request_cert_old(config: &crate::config::Config) -> Result<(), Error> { // Use DirectoryUrl::LetsEncrypStaging for dev/testing. let url = if config.opts.development { DirectoryUrl::LetsEncryptStaging @@ -189,3 +193,133 @@ fn request_cert(config: &crate::config::Config) -> Result<(), Error> { tracing::info!("HTTPS init Success!"); Ok(()) } + +async fn request_cert(config: &crate::config::Config) -> AtomicServerResult<()> { + // Create a new account. This will generate a fresh ECDSA key for you. + // Alternatively, restore an account from serialized credentials by + // using `Account::from_credentials()`. + + let url = if config.opts.development { + instant_acme::LetsEncrypt::Staging.url() + } else { + instant_acme::LetsEncrypt::Production.url() + }; + + let account = instant_acme::Account::create( + &instant_acme::NewAccount { + contact: &[&config.opts.email.clone().expect( + "No email set - required for HTTPS certificate initialization with LetsEncrypt", + )], + terms_of_service_agreed: true, + only_return_existing: false, + }, + url, + ) + .await + .map_err(|e| format!("Failed to create account: {}", e))?; + + // Create the ACME order based on the given domain names. + // Note that this only needs an `&Account`, so the library will let you + // process multiple orders in parallel for a single account. + + let identifier = instant_acme::Identifier::Dns(config.opts.domain.clone()); + let (mut order, state) = account + .new_order(&instant_acme::NewOrder { + identifiers: &[identifier], + }) + .await + .unwrap(); + + tracing::info!("order state: {:#?}", state); + assert!(matches!(state.status, instant_acme::OrderStatus::Pending)); + + // Pick the desired challenge type and prepare the response. + + let authorizations = order.authorizations(&state.authorizations).await.unwrap(); + let mut challenges = Vec::with_capacity(authorizations.len()); + for authz in &authorizations { + match authz.status { + instant_acme::AuthorizationStatus::Pending => {} + instant_acme::AuthorizationStatus::Valid => continue, + _ => todo!(), + } + + let challenge = authz + .challenges + .iter() + .find(|c| c.r#type == instant_acme::ChallengeType::Http01) + .ok_or("no Http01 challenge found")?; + + let instant_acme::Identifier::Dns(identifier) = &authz.identifier; + + println!("Please set the following DNS record then press any key:"); + println!( + "_acme-challenge.{} IN TXT {}", + identifier, + order.key_authorization(challenge).dns_value() + ); + std::io::stdin().read_line(&mut String::new()).unwrap(); + + challenges.push((identifier, &challenge.url)); + } + + // Let the server know we're ready to accept the challenges. + for (_, url) in &challenges { + order.set_challenge_ready(url).await.unwrap(); + } + + // Exponentially back off until the order becomes ready or invalid. + let mut tries = 1u8; + let mut delay = std::time::Duration::from_millis(250); + let state = loop { + actix::clock::sleep(delay).await; + let state = order.state().await.unwrap(); + if let instant_acme::OrderStatus::Ready | instant_acme::OrderStatus::Invalid = state.status + { + tracing::info!("order state: {:#?}", state); + break state; + } + + delay *= 2; + tries += 1; + match tries < 5 { + true => info!(?state, tries, "order is not ready, waiting {delay:?}"), + false => { + return Err("order is not ready".into()); + } + } + }; + + if state.status == OrderStatus::Invalid { + return Err("order is invalid".into()); + } + + let mut names = Vec::with_capacity(challenges.len()); + for (identifier, _) in challenges { + names.push(identifier.to_owned()); + } + + // If the order is ready, we can provision the certificate. + // Use the rcgen library to create a Certificate Signing Request. + + let mut params = rcgen::CertificateParams::new(names.clone()); + params.distinguished_name = rcgen::DistinguishedName::new(); + let cert = rcgen::Certificate::from_params(params).unwrap(); + let csr = cert.serialize_request_der().map_err(|e| e.to_string())?; + + // Finalize the order and print certificate chain, private key and account credentials. + + let cert_chain_pem = order.finalize(&csr, &state.finalize).await.unwrap(); + info!("certficate chain:\n\n{}", cert_chain_pem,); + info!("private key:\n\n{}", cert.serialize_private_key_pem()); + info!( + "account credentials:\n\n{}", + serde_json::to_string_pretty(&account.credentials()).unwrap() + ); + fs::write(&config.cert_path, cert_chain_pem).expect("Unable to write cert file"); + fs::write(&config.key_path, cert.serialize_private_key_pem()) + .expect("Unable to write key file"); + set_certs_created_at_file(config); + + Ok(()) +} From 03623d3faef7b7955938e2881c063cd057e30bc6 Mon Sep 17 00:00:00 2001 From: Joep Meindertsma Date: Wed, 8 Feb 2023 19:30:08 +0100 Subject: [PATCH 2/8] Remove openssl #192 --- CONTRIBUTE.md | 4 +- dockerfile | 2 - server/src/bin.rs | 2 - server/src/errors.rs | 11 -- server/src/https.rs | 211 +++++++++++++++++++++++++ server/src/https_init.rs | 325 --------------------------------------- server/src/lib.rs | 2 - server/src/serve.rs | 3 +- 8 files changed, 214 insertions(+), 346 deletions(-) delete mode 100644 server/src/https_init.rs diff --git a/CONTRIBUTE.md b/CONTRIBUTE.md index b45401d74..e30751f1b 100644 --- a/CONTRIBUTE.md +++ b/CONTRIBUTE.md @@ -53,7 +53,7 @@ If you want to build `atomic-server` for some other target (e.g. building for li ```sh cargo install cross # make sure docker is running! -cross build --target x86_64-unknown-linux-musl +cross build --target x86_64-unknown-linux-musl --bin atomic-server --release ``` ## IDE setup (VSCode) @@ -230,7 +230,7 @@ or: or do it manually: 1. `cd server` -1. `cargo build --release --target x86_64-unknown-linux-gnu` +1. `cargo build --release --target x86_64-unknown-linux-musl --bin atomic-server` (if it fails, use cross, see above) 1. `scp ../target/x86_64-unknown-linux-gnu/release/atomic-server atomic:~/atomic/server/atomic-server-v0.{version}` 1. `ssh atomic` (@joepio manages server) 2. `service atomic restart` diff --git a/dockerfile b/dockerfile index b25b49b31..977f54edd 100644 --- a/dockerfile +++ b/dockerfile @@ -1,8 +1,6 @@ FROM frolvlad/alpine-rust as builder WORKDIR /app COPY . . -RUN apk add --no-cache openssl -RUN apk add --no-cache openssl-dev RUN cargo build --release --bin atomic-server # We only need a small runtime for this step, but make sure glibc is installed diff --git a/server/src/bin.rs b/server/src/bin.rs index 7297d484d..835ded185 100644 --- a/server/src/bin.rs +++ b/server/src/bin.rs @@ -11,8 +11,6 @@ mod handlers; mod helpers; #[cfg(feature = "https")] mod https; -#[cfg(feature = "https_init")] -mod https_init; mod jsonerrors; #[cfg(feature = "process-management")] mod process; diff --git a/server/src/errors.rs b/server/src/errors.rs index 41e3db4a6..b84b7fe2f 100644 --- a/server/src/errors.rs +++ b/server/src/errors.rs @@ -166,17 +166,6 @@ impl From for AtomicServerError { } } -#[cfg(feature = "https_init")] -impl From for AtomicServerError { - fn from(error: acme_lib::Error) -> Self { - AtomicServerError { - message: error.to_string(), - error_type: AppErrorType::Other, - error_resource: None, - } - } -} - impl From for AtomicServerError { fn from(error: actix_web::Error) -> Self { AtomicServerError { diff --git a/server/src/https.rs b/server/src/https.rs index 0dc614b8b..7be3f2b88 100644 --- a/server/src/https.rs +++ b/server/src/https.rs @@ -79,3 +79,214 @@ pub fn should_renew_certs_check(config: &crate::config::Config) -> bool { }; expired } + +use actix_web::{App, HttpServer}; +use instant_acme::OrderStatus; +use tracing::info; + +use std::sync::mpsc; + +/// Starts an HTTP Actix server for HTTPS certificate initialization +pub async fn cert_init_server(config: &crate::config::Config) -> AtomicServerResult<()> { + let address = format!("{}:{}", config.opts.ip, config.opts.port); + tracing::warn!("Server temporarily running in HTTP mode at {}, running Let's Encrypt Certificate initialization...", address); + + if config.opts.port != 80 { + tracing::warn!( + "HTTP port is {}, not 80. Should be 80 in most cases during LetsEncrypt setup.", + config.opts.port + ); + } + + let mut well_known_folder = config.static_path.clone(); + well_known_folder.push("well-known"); + fs::create_dir_all(&well_known_folder)?; + + let (tx, rx) = mpsc::channel(); + + let address_clone = address.clone(); + + std::thread::spawn(move || { + actix_web::rt::System::new().block_on(async move { + let init_server = HttpServer::new(move || { + App::new().service( + actix_files::Files::new("/.well-known", well_known_folder.clone()) + .show_files_listing(), + ) + }); + + let running_server = init_server.bind(&address_clone)?.run(); + + tx.send(running_server.handle()) + .expect("Error sending handle during HTTPS init."); + + running_server.await + }) + }); + + let handle = rx + .recv() + .map_err(|e| format!("Error receiving handle during HTTPS init. {}", e))?; + + let agent = ureq::builder() + .timeout(std::time::Duration::from_secs(2)) + .build(); + + let well_known_url = format!("http://{}/.well-known/", &config.opts.domain); + tracing::info!("Testing availability of {}", &well_known_url); + let resp = agent.get(&well_known_url).call().map_err(|e| { + format!( + "Unable to send request for Let's Encrypt initialization. {}", + e + ) + })?; + if resp.status() != 200 { + return Err( + "Server for HTTP initialization not available, returning a non-200 status code".into(), + ); + } else { + tracing::info!("Server for HTTP initialization running correctly"); + } + + request_cert(config) + .await + .map_err(|e| format!("Certification init failed: {}", e))?; + tracing::warn!("HTTPS TLS Cert init sucesful! Stopping HTTP server, starting HTTPS..."); + handle.stop(true).await; + Ok(()) +} + +async fn request_cert(config: &crate::config::Config) -> AtomicServerResult<()> { + // Create a new account. This will generate a fresh ECDSA key for you. + // Alternatively, restore an account from serialized credentials by + // using `Account::from_credentials()`. + + let url = if config.opts.development { + instant_acme::LetsEncrypt::Staging.url() + } else { + instant_acme::LetsEncrypt::Production.url() + }; + + let email = + config.opts.email.clone().expect( + "No email set - required for HTTPS certificate initialization with LetsEncrypt", + ); + + info!("Creating LetsEncrypt account with email {}", email); + + let account = instant_acme::Account::create( + &instant_acme::NewAccount { + contact: &[&email], + terms_of_service_agreed: true, + only_return_existing: false, + }, + url, + ) + .await + .map_err(|e| format!("Failed to create account: {}", e))?; + + // Create the ACME order based on the given domain names. + // Note that this only needs an `&Account`, so the library will let you + // process multiple orders in parallel for a single account. + + let identifier = instant_acme::Identifier::Dns(config.opts.domain.clone()); + let (mut order, state) = account + .new_order(&instant_acme::NewOrder { + identifiers: &[identifier], + }) + .await + .unwrap(); + + tracing::info!("order state: {:#?}", state); + assert!(matches!(state.status, instant_acme::OrderStatus::Pending)); + + // Pick the desired challenge type and prepare the response. + + let authorizations = order.authorizations(&state.authorizations).await.unwrap(); + let mut challenges = Vec::with_capacity(authorizations.len()); + for authz in &authorizations { + match authz.status { + instant_acme::AuthorizationStatus::Pending => {} + instant_acme::AuthorizationStatus::Valid => continue, + _ => todo!(), + } + + let challenge = authz + .challenges + .iter() + .find(|c| c.r#type == instant_acme::ChallengeType::Http01) + .ok_or("no Http01 challenge found")?; + + let instant_acme::Identifier::Dns(identifier) = &authz.identifier; + + println!("Please set the following DNS record then press any key:"); + println!( + "_acme-challenge.{} IN TXT {}", + identifier, + order.key_authorization(challenge).dns_value() + ); + std::io::stdin().read_line(&mut String::new()).unwrap(); + + challenges.push((identifier, &challenge.url)); + } + + // Let the server know we're ready to accept the challenges. + for (_, url) in &challenges { + order.set_challenge_ready(url).await.unwrap(); + } + + // Exponentially back off until the order becomes ready or invalid. + let mut tries = 1u8; + let mut delay = std::time::Duration::from_millis(250); + let state = loop { + actix::clock::sleep(delay).await; + let state = order.state().await.unwrap(); + if let instant_acme::OrderStatus::Ready | instant_acme::OrderStatus::Invalid = state.status + { + tracing::info!("order state: {:#?}", state); + break state; + } + + delay *= 2; + tries += 1; + match tries < 5 { + true => info!(?state, tries, "order is not ready, waiting {delay:?}"), + false => { + return Err("order is not ready".into()); + } + } + }; + + if state.status == OrderStatus::Invalid { + return Err("order is invalid".into()); + } + + let mut names = Vec::with_capacity(challenges.len()); + for (identifier, _) in challenges { + names.push(identifier.to_owned()); + } + + // If the order is ready, we can provision the certificate. + // Use the rcgen library to create a Certificate Signing Request. + + let mut params = rcgen::CertificateParams::new(names.clone()); + params.distinguished_name = rcgen::DistinguishedName::new(); + let cert = rcgen::Certificate::from_params(params).unwrap(); + let csr = cert.serialize_request_der().map_err(|e| e.to_string())?; + + // Finalize the order and print certificate chain, private key and account credentials. + + let cert_chain_pem = order.finalize(&csr, &state.finalize).await.unwrap(); + info!("certficate chain:\n\n{}", cert_chain_pem,); + info!("private key:\n\n{}", cert.serialize_private_key_pem()); + info!( + "account credentials:\n\n{}", + serde_json::to_string_pretty(&account.credentials()).unwrap() + ); + fs::write(&config.cert_path, cert_chain_pem).expect("Unable to write cert file"); + fs::write(&config.key_path, cert.serialize_private_key_pem()) + .expect("Unable to write key file"); + set_certs_created_at_file(config); + + Ok(()) +} diff --git a/server/src/https_init.rs b/server/src/https_init.rs deleted file mode 100644 index 07fa92227..000000000 --- a/server/src/https_init.rs +++ /dev/null @@ -1,325 +0,0 @@ -//! Everything required for setting up HTTPS. -//! Calls the Let's Encrypt API to get a certificate. - -use acme_lib::create_p384_key; -use acme_lib::persist::FilePersist; -use acme_lib::{Directory, DirectoryUrl, Error}; -use actix_web::{App, HttpServer}; -use instant_acme::OrderStatus; -use tracing::{error, info}; - -use std::sync::mpsc; -use std::{ - fs::{self}, - path::PathBuf, -}; - -use crate::errors::AtomicServerResult; -use crate::https::set_certs_created_at_file; - -/// Starts an HTTP Actix server for HTTPS certificate initialization -pub async fn cert_init_server(config: &crate::config::Config) -> AtomicServerResult<()> { - let address = format!("{}:{}", config.opts.ip, config.opts.port); - tracing::warn!("Server temporarily running in HTTP mode at {}, running Let's Encrypt Certificate initialization...", address); - - let mut well_known_folder = config.static_path.clone(); - well_known_folder.push("well-known"); - fs::create_dir_all(&well_known_folder)?; - - let (tx, rx) = mpsc::channel(); - - let address_clone = address.clone(); - - std::thread::spawn(move || { - actix_web::rt::System::new().block_on(async move { - let init_server = HttpServer::new(move || { - App::new().service( - actix_files::Files::new("/.well-known", well_known_folder.clone()) - .show_files_listing(), - ) - }); - - let running_server = init_server.bind(&address_clone)?.run(); - - tx.send(running_server.handle()) - .expect("Error sending handle during HTTPS init"); - - running_server.await - }) - }); - - let handle = rx.recv().expect("Error receiving handle during HTTPS init"); - - let agent = ureq::builder() - .timeout(std::time::Duration::from_secs(2)) - .build(); - - let well_known_url = format!("http://{}/.well-known/", &config.opts.domain); - tracing::info!("Testing availability of {}", &well_known_url); - let resp = agent - .get(&well_known_url) - .call() - .expect("Unable to send request for Let's Encrypt initialization"); - if resp.status() != 200 { - return Err( - "Server for HTTP initialization not available, returning a non-200 status code".into(), - ); - } else { - tracing::info!("Server for HTTP initialization running correctly"); - } - - request_cert(config) - .await - .map_err(|e| format!("Certification init failed: {}", e))?; - tracing::warn!("HTTPS TLS Cert init sucesful! Stopping HTTP server, starting HTTPS..."); - handle.stop(true).await; - Ok(()) -} - -/// Writes keys to disk using LetsEncrypt -fn request_cert_old(config: &crate::config::Config) -> Result<(), Error> { - // Use DirectoryUrl::LetsEncrypStaging for dev/testing. - let url = if config.opts.development { - DirectoryUrl::LetsEncryptStaging - } else { - DirectoryUrl::LetsEncrypt - }; - - fs::create_dir_all(PathBuf::from(&config.https_path))?; - - // Save/load keys and certificates to current dir. - let persist = FilePersist::new(&config.https_path); - - // Create a directory entrypoint. - let dir = Directory::from_url(persist, url)?; - - // Reads the private account key from persistence, or - // creates a new one before accessing the API to establish - // that it's there. - let email = config - .opts - .email - .clone() - .expect("ATOMIC_EMAIL must be set for HTTPS init"); - tracing::info!("Requesting Let's Encrypt account with {}", email); - let acc = dir.account(&email)?; - - // Order a new TLS certificate for a domain. - let mut ord_new = acc.new_order(&config.opts.domain, &[])?; - - // If the ownership of the domain(s) have already been - // authorized in a previous order, you might be able to - // skip validation. The ACME API provider decides. - let ord_csr = loop { - // are we done? - if let Some(ord_csr) = ord_new.confirm_validations() { - break ord_csr; - } - - // Get the possible authorizations (for a single domain - // this will only be one element). - let auths = ord_new.authorizations()?; - - // For HTTP, the challenge is a text file that needs to - // be placed in your web server's root: - // - // /var/www/.well-known/acme-challenge/ - // - // The important thing is that it's accessible over the - // web for the domain(s) you are trying to get a - // certificate for: - // - // http://mydomain.io/.well-known/acme-challenge/ - let chall = auths[0].http_challenge(); - - // The token is the filename. - let token = chall.http_token(); - - let formatted_path = format!("well-known/acme-challenge/{}", token); - let mut challenge_path = config.static_path.clone(); - challenge_path.push(formatted_path); - - // The proof is the contents of the file - let proof = chall.http_proof(); - - tracing::info!("Writing ACME challange to {:?}", challenge_path); - - fs::create_dir_all( - PathBuf::from(&challenge_path) - .parent() - .expect("Could not find parent folder"), - ) - .expect("Unable to create dirs"); - - fs::write(challenge_path, proof).expect("Unable to write file"); - - // Here you must do "something" to place - // the file/contents in the correct place. - // update_my_web_server(&path, &proof); - - // After the file is accessible from the web, the calls - // this to tell the ACME API to start checking the - // existence of the proof. - // - // The order at ACME will change status to either - // confirm ownership of the domain, or fail due to the - // not finding the proof. To see the change, we poll - // the API with 5000 milliseconds wait between. - chall.validate(5000)?; - - // Update the state against the ACME API. - ord_new.refresh()?; - }; - - // Ownership is proven. Create a private key for - // the certificate. These are provided for convenience, you - // can provide your own keypair instead if you want. - let pkey_pri = create_p384_key(); - - // Submit the CSR. This causes the ACME provider to enter a - // state of "processing" that must be polled until the - // certificate is either issued or rejected. Again we poll - // for the status change. - let ord_cert = ord_csr.finalize_pkey(pkey_pri, 5000)?; - - // Now download the certificate. Also stores the cert in - // the persistence. - tracing::info!("Downloading certificate..."); - let cert = ord_cert.download_and_save_cert()?; - - fs::write(&config.cert_path, cert.certificate()).expect("Unable to write file"); - fs::write(&config.key_path, cert.private_key()).expect("Unable to write file"); - set_certs_created_at_file(config); - tracing::info!("HTTPS init Success!"); - Ok(()) -} - -async fn request_cert(config: &crate::config::Config) -> AtomicServerResult<()> { - // Create a new account. This will generate a fresh ECDSA key for you. - // Alternatively, restore an account from serialized credentials by - // using `Account::from_credentials()`. - - let url = if config.opts.development { - instant_acme::LetsEncrypt::Staging.url() - } else { - instant_acme::LetsEncrypt::Production.url() - }; - - let account = instant_acme::Account::create( - &instant_acme::NewAccount { - contact: &[&config.opts.email.clone().expect( - "No email set - required for HTTPS certificate initialization with LetsEncrypt", - )], - terms_of_service_agreed: true, - only_return_existing: false, - }, - url, - ) - .await - .map_err(|e| format!("Failed to create account: {}", e))?; - - // Create the ACME order based on the given domain names. - // Note that this only needs an `&Account`, so the library will let you - // process multiple orders in parallel for a single account. - - let identifier = instant_acme::Identifier::Dns(config.opts.domain.clone()); - let (mut order, state) = account - .new_order(&instant_acme::NewOrder { - identifiers: &[identifier], - }) - .await - .unwrap(); - - tracing::info!("order state: {:#?}", state); - assert!(matches!(state.status, instant_acme::OrderStatus::Pending)); - - // Pick the desired challenge type and prepare the response. - - let authorizations = order.authorizations(&state.authorizations).await.unwrap(); - let mut challenges = Vec::with_capacity(authorizations.len()); - for authz in &authorizations { - match authz.status { - instant_acme::AuthorizationStatus::Pending => {} - instant_acme::AuthorizationStatus::Valid => continue, - _ => todo!(), - } - - let challenge = authz - .challenges - .iter() - .find(|c| c.r#type == instant_acme::ChallengeType::Http01) - .ok_or("no Http01 challenge found")?; - - let instant_acme::Identifier::Dns(identifier) = &authz.identifier; - - println!("Please set the following DNS record then press any key:"); - println!( - "_acme-challenge.{} IN TXT {}", - identifier, - order.key_authorization(challenge).dns_value() - ); - std::io::stdin().read_line(&mut String::new()).unwrap(); - - challenges.push((identifier, &challenge.url)); - } - - // Let the server know we're ready to accept the challenges. - for (_, url) in &challenges { - order.set_challenge_ready(url).await.unwrap(); - } - - // Exponentially back off until the order becomes ready or invalid. - let mut tries = 1u8; - let mut delay = std::time::Duration::from_millis(250); - let state = loop { - actix::clock::sleep(delay).await; - let state = order.state().await.unwrap(); - if let instant_acme::OrderStatus::Ready | instant_acme::OrderStatus::Invalid = state.status - { - tracing::info!("order state: {:#?}", state); - break state; - } - - delay *= 2; - tries += 1; - match tries < 5 { - true => info!(?state, tries, "order is not ready, waiting {delay:?}"), - false => { - return Err("order is not ready".into()); - } - } - }; - - if state.status == OrderStatus::Invalid { - return Err("order is invalid".into()); - } - - let mut names = Vec::with_capacity(challenges.len()); - for (identifier, _) in challenges { - names.push(identifier.to_owned()); - } - - // If the order is ready, we can provision the certificate. - // Use the rcgen library to create a Certificate Signing Request. - - let mut params = rcgen::CertificateParams::new(names.clone()); - params.distinguished_name = rcgen::DistinguishedName::new(); - let cert = rcgen::Certificate::from_params(params).unwrap(); - let csr = cert.serialize_request_der().map_err(|e| e.to_string())?; - - // Finalize the order and print certificate chain, private key and account credentials. - - let cert_chain_pem = order.finalize(&csr, &state.finalize).await.unwrap(); - info!("certficate chain:\n\n{}", cert_chain_pem,); - info!("private key:\n\n{}", cert.serialize_private_key_pem()); - info!( - "account credentials:\n\n{}", - serde_json::to_string_pretty(&account.credentials()).unwrap() - ); - fs::write(&config.cert_path, cert_chain_pem).expect("Unable to write cert file"); - fs::write(&config.key_path, cert.serialize_private_key_pem()) - .expect("Unable to write key file"); - set_certs_created_at_file(config); - - Ok(()) -} diff --git a/server/src/lib.rs b/server/src/lib.rs index fb04dcfe3..b3c5babfd 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -13,8 +13,6 @@ mod handlers; mod helpers; #[cfg(feature = "https")] mod https; -#[cfg(feature = "https_init")] -mod https_init; mod jsonerrors; #[cfg(feature = "process-management")] mod process; diff --git a/server/src/serve.rs b/server/src/serve.rs index b36f55b0c..b6a9f663e 100644 --- a/server/src/serve.rs +++ b/server/src/serve.rs @@ -71,10 +71,9 @@ pub async fn serve(config: crate::config::Config) -> AtomicServerResult<()> { #[cfg(feature = "https")] { // If there is no certificate file, or the certs are too old, start HTTPS initialization - #[cfg(feature = "https_init")] { if crate::https::should_renew_certs_check(&config) { - crate::https_init::cert_init_server(&config).await?; + crate::https::cert_init_server(&config).await?; } } let https_config = crate::https::get_https_config(&config) From 5aa187f1238fc72ec1562415fe50249d71961196 Mon Sep 17 00:00:00 2001 From: Joep Meindertsma Date: Wed, 8 Feb 2023 21:43:15 +0100 Subject: [PATCH 3/8] use dns check --- server/src/https.rs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/server/src/https.rs b/server/src/https.rs index 7be3f2b88..68d22eb14 100644 --- a/server/src/https.rs +++ b/server/src/https.rs @@ -157,6 +157,10 @@ pub async fn cert_init_server(config: &crate::config::Config) -> AtomicServerRes } async fn request_cert(config: &crate::config::Config) -> AtomicServerResult<()> { + let use_wildcard = false; + + fs::create_dir_all(PathBuf::from(&config.https_path))?; + // Create a new account. This will generate a fresh ECDSA key for you. // Alternatively, restore an account from serialized credentials by // using `Account::from_credentials()`. @@ -176,7 +180,7 @@ async fn request_cert(config: &crate::config::Config) -> AtomicServerResult<()> let account = instant_acme::Account::create( &instant_acme::NewAccount { - contact: &[&email], + contact: &[&format!("mailto:{}", email)], terms_of_service_agreed: true, only_return_existing: false, }, @@ -189,7 +193,11 @@ async fn request_cert(config: &crate::config::Config) -> AtomicServerResult<()> // Note that this only needs an `&Account`, so the library will let you // process multiple orders in parallel for a single account. - let identifier = instant_acme::Identifier::Dns(config.opts.domain.clone()); + let mut domain = config.opts.domain.clone(); + if use_wildcard { + domain = format!("*.{}", domain); + } + let identifier = instant_acme::Identifier::Dns(domain); let (mut order, state) = account .new_order(&instant_acme::NewOrder { identifiers: &[identifier], @@ -214,8 +222,8 @@ async fn request_cert(config: &crate::config::Config) -> AtomicServerResult<()> let challenge = authz .challenges .iter() - .find(|c| c.r#type == instant_acme::ChallengeType::Http01) - .ok_or("no Http01 challenge found")?; + .find(|c| c.r#type == instant_acme::ChallengeType::Dns01) + .ok_or("no Dns01 challenge found")?; let instant_acme::Identifier::Dns(identifier) = &authz.identifier; From 563d28e0c7171ff3658d530b7c6ed6c44a0e4225 Mon Sep 17 00:00:00 2001 From: Joep Meindertsma Date: Thu, 9 Feb 2023 22:31:03 +0100 Subject: [PATCH 4/8] Add staging CI #589 Try on push Try develop #192 https setup docs try again add environments fix yml Try nested job try again Require secrets remote_host fix inputs Add remote host for production --- .github/workflows/deploy_production.yml | 14 ++++++++++++++ .github/workflows/deploy_staging.yml | 14 ++++++++++++++ .github/workflows/deployment.yml | 21 +++++++++++++++++---- server/default.env | 15 +++++++++------ server/src/https.rs | 2 +- 5 files changed, 55 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/deploy_production.yml create mode 100644 .github/workflows/deploy_staging.yml diff --git a/.github/workflows/deploy_production.yml b/.github/workflows/deploy_production.yml new file mode 100644 index 000000000..012d695c8 --- /dev/null +++ b/.github/workflows/deploy_production.yml @@ -0,0 +1,14 @@ +name: Deployment Staging + +on: + workflow_dispatch: + push: + branches: + - 'master' +jobs: + deploy-staging: + uses: './.github/workflows/deployment.yml' + with: + environment: production + remote_host: atomicdata.dev + secrets: inherit diff --git a/.github/workflows/deploy_staging.yml b/.github/workflows/deploy_staging.yml new file mode 100644 index 000000000..102b53065 --- /dev/null +++ b/.github/workflows/deploy_staging.yml @@ -0,0 +1,14 @@ +name: Deployment Staging + +on: + workflow_dispatch: + push: + branches: + - 'develop' +jobs: + deploy-staging: + uses: './.github/workflows/deployment.yml' + with: + environment: staging + remote_host: staging.atomicdata.dev + secrets: inherit diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml index c3dd90692..ad892d9f5 100644 --- a/.github/workflows/deployment.yml +++ b/.github/workflows/deployment.yml @@ -1,10 +1,23 @@ -name: Deployment +name: Deployment (re-use) on: - workflow_dispatch: + workflow_call: + inputs: + environment: + required: true + type: string + remote_host: + required: true + type: string + secrets: + REMOTE_USER: + required: true + SSH_PRIVATE_KEY: + required: true jobs: deploy: + environment: ${{ inputs.environment }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -25,13 +38,13 @@ jobs: SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} ARGS: "-rltgoDzvO" SOURCE: "target/x86_64-unknown-linux-musl/release/atomic-server" - REMOTE_HOST: ${{ secrets.REMOTE_HOST }} + REMOTE_HOST: ${{ inputs.remote_host }} REMOTE_USER: ${{ secrets.REMOTE_USER }} TARGET: ~/ - name: executing remote ssh commands using ssh key uses: appleboy/ssh-action@master with: - host: ${{ secrets.REMOTE_HOST }} + host: ${{ inputs.remote_host }} username: ${{ secrets.REMOTE_USER }} key: ${{ secrets.SSH_PRIVATE_KEY }} script: | diff --git a/server/default.env b/server/default.env index c135aeca1..29c9e21dc 100644 --- a/server/default.env +++ b/server/default.env @@ -6,6 +6,15 @@ # domain name where the server is hosted, including subdoman, without schema (http) and port # ATOMIC_DOMAIN=localhost +# Port where it's running. Make sure you have the rights to run at this port. +# Both an HTTPS and an HTTP port can be specified. + +# Set this to 80 if you're running this publicly +# ATOMIC_PORT=9883 + +# Set this to 443 if you're running this publicly +# ATOMIC_PORT_HTTPS=9884 + # Whether you want to enable HTTPS. If set to true, the server will initialize Certification process on start using Let'sEncrypt. You'll need to set ATOMIC_EMAIL for this. # ATOMIC_HTTPS=false @@ -24,12 +33,6 @@ # Path to where your config will be stored. # ATOMIC_CONFIG_FILE_PATH="/Users/your_home_folder/.config/atomic/config.toml" -# Port where it's running. Make sure you have the rights to run at this port. -# Both an HTTPS and an HTTP port can be specified. -# Both are necesssary during CERT_INIT -# ATOMIC_PORT=9883 -# ATOMIC_PORT_HTTPS=9884 - # Local IP where it's running. Use :: if you want it to be public. # ATOMIC_IP=:: diff --git a/server/src/https.rs b/server/src/https.rs index 68d22eb14..c9eb7a978 100644 --- a/server/src/https.rs +++ b/server/src/https.rs @@ -151,7 +151,7 @@ pub async fn cert_init_server(config: &crate::config::Config) -> AtomicServerRes request_cert(config) .await .map_err(|e| format!("Certification init failed: {}", e))?; - tracing::warn!("HTTPS TLS Cert init sucesful! Stopping HTTP server, starting HTTPS..."); + tracing::warn!("HTTPS TLS Cert init successful! Stopping HTTP server, starting HTTPS..."); handle.stop(true).await; Ok(()) } From 4e571abcd9aaa9d146a7ad6096547e1295724fcf Mon Sep 17 00:00:00 2001 From: Joep Meindertsma Date: Thu, 9 Feb 2023 22:47:39 +0100 Subject: [PATCH 5/8] Add systemd instructions to readme #271 --- CHANGELOG.md | 2 ++ server/README.md | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cd87976e8..9687a0939 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,8 @@ See [STATUS.md](server/STATUS.md) to learn more about which features will remain - Include JSON-AD in initial response, speed up first render #511 - Remove feature to index external RDF files and search them #579 - Replace `acme_lib` with `instant-acme`, drop OpenSSL dependency #192 +- Add staging environment #588 +- Add systemd instructions to readme #271 ## [v0.34.0] - 2022-10-31 diff --git a/server/README.md b/server/README.md index c52f61efc..fa33903cb 100644 --- a/server/README.md +++ b/server/README.md @@ -46,6 +46,7 @@ https://user-images.githubusercontent.com/2183313/139728539-d69b899f-6f9b-44cb-a - [Running using a tunneling service (easy mode)](#running-using-a-tunneling-service-easy-mode) - [HTTPS Setup on a VPS (static IP required)](#https-setup-on-a-vps-static-ip-required) - [HTTPS Setup using external HTTPS proxy](#https-setup-using-external-https-proxy) + - [Using `systemd` to run Atomic-Server as a service](#using-systemd-to-run-atomic-server-as-a-service) - [Usage](#usage) - [Using Atomic-Server with the browser GUI](#using-atomic-server-with-the-browser-gui) - [Use `atomic-cli` as client](#use-atomic-cli-as-client) @@ -184,6 +185,48 @@ ATOMIC_HTTPS=false ATOMIC_SERVER_URL=https://example.com ``` +### Using `systemd` to run Atomic-Server as a service + +In Unix operating systems, you can use `systemd` to manage a + +Create a service: + +```sh +vim /etc/systemd/system/atomic.service +``` + +Paste this: +``` +[Unit] +Description=Atomic-Server +#After=network.targetdd +StartLimitIntervalSec=0[Service] + +[Service] +Type=simple +Restart=always +RestartSec=1 +User=root +ExecStart=/root/atomic-server +WorkingDirectory=/root/ +EnvironmentFil=/root/.env + +[Install] +WantedBy=multi-user.target +``` + +```sh +# start service +systemctl start atomic +# check status +systemctl status atomic +# restart +systemctl restart atomic +# logs +journalctl -u atomic.service +# logs, since one hour, follow +journalctl -u atomic.service --since "1 hour ago" -f +``` ## Usage There are three ways to interact with this server: From 0bb3f55eb828c9c603c58b48b777b06031351f6c Mon Sep 17 00:00:00 2001 From: Joep Meindertsma Date: Fri, 10 Feb 2023 09:19:32 +0100 Subject: [PATCH 6/8] Improve readme --- README.md | 4 ++-- server/README.md | 30 +++++++++++++++--------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index b24ddb1a2..cbcc58c6c 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ _Status: Beta. [Status](server/STATUS.md) specifies which features are stable. [ **Atomic-server is a graph database server for storing and sharing [Atomic Data](https://docs.atomicdata.dev/). Demo on [atomicdata.dev](https://atomicdata.dev)** -- 🚀 **Fast** (1ms median response time on my laptop), powered by [actix-web](https://github.com/actix/actix-web) and [sled](https://github.com/spacejam/sled) +- 🚀 **Fast** (less than 1ms median response time on my laptop), powered by [actix-web](https://github.com/actix/actix-web) and [sled](https://github.com/spacejam/sled) - ðŸŠķ **Lightweight** (8MB download, no runtime dependencies) - ðŸ’ŧ **Runs everywhere** (linux, windows, mac, arm) - ⚛ïļ **Dynamic schema validation** / type checking using [Atomic Schema](https://docs.atomicdata.dev/schema/intro.html). @@ -32,7 +32,7 @@ Demo on [atomicdata.dev](https://atomicdata.dev)** - ðŸ“ē **Invite and sharing system** with [Atomic Invites](https://docs.atomicdata.dev/invitations.html) - 📂 **File management**: Upload, download and preview attachments. - ðŸ–Ĩïļ **Desktop app**: Easy desktop installation, with status bar icon, powered by [tauri](https://github.com/tauri-apps/tauri/). - +- 📚 **Libraries**: [Javascript / Typescript](https://www.npmjs.com/package/@tomic/lib), [React](https://www.npmjs.com/package/@tomic/react), [Svelte](https://www.npmjs.com/package/@tomic/svelte) Powered by Rust, [atomic-lib](https://crates.io/crates/atomic-lib) and [more](Cargo.toml). [→ Read more](server/README.md) diff --git a/server/README.md b/server/README.md index fa33903cb..283e70c4e 100644 --- a/server/README.md +++ b/server/README.md @@ -11,7 +11,7 @@ _Status: alpha. [Breaking changes](../CHANGELOG.md) are expected until 1.0._ Demo on [atomicdata.dev](https://atomicdata.dev)** -- 🚀 **Fast** (1ms median response time on my laptop), powered by [actix-web](https://github.com/actix/actix-web) and [sled](https://github.com/spacejam/sled) +- 🚀 **Fast** (less than 1ms median response time on my laptop), powered by [actix-web](https://github.com/actix/actix-web) and [sled](https://github.com/spacejam/sled) - ðŸŠķ **Lightweight** (8MB download, no runtime dependencies) - ðŸ’ŧ **Runs everywhere** (linux, windows, mac, arm) - ⚛ïļ **Dynamic schema validation** / type checking using [Atomic Schema](https://docs.atomicdata.dev/schema/intro.html). @@ -26,6 +26,7 @@ Demo on [atomicdata.dev](https://atomicdata.dev)** - ðŸ“ē **Invite and sharing system** with [Atomic Invites](https://docs.atomicdata.dev/invitations.html) - 📂 **File management**: Upload, download and preview attachments. - ðŸ–Ĩïļ **Desktop app**: Easy desktop installation, with status bar icon, powered by [tauri](https://github.com/tauri-apps/tauri/). +- 📚 **Libraries**: [Javascript / Typescript](https://www.npmjs.com/package/@tomic/lib), [React](https://www.npmjs.com/package/@tomic/react), [Svelte](https://www.npmjs.com/package/@tomic/svelte) Powered by Rust, [atomic-lib](https://crates.io/crates/atomic-lib) and [more](Cargo.toml). @@ -64,18 +65,19 @@ https://user-images.githubusercontent.com/2183313/139728539-d69b899f-6f9b-44cb-a ## When should you use this -- You want a powerful, lightweight, fast and easy to use **CMS** with editors, modelling capabilities and an intuitive API +- You want a powerful, lightweight, fast and easy to use **CMS or database** with live updates, editors, modelling capabilities and an intuitive API +- You want to build a webapplication, and like working with using [React](https://github.com/atomicdata-dev/atomic-data-browser) or [Svelte](https://github.com/atomicdata-dev/atomic-svelte). - You want to make (high-value) **datasets as easily accessible as possible** - You want to specify and share a **common vocabulary** / ontology / schema for some specific domain or dataset. Example classes [here](https://atomicdata.dev/classes). - You want to use and **share linked data**, but don't want to deal with most of [the complexities of RDF](https://docs.atomicdata.dev/interoperability/rdf.html), SPARQL, Triple Stores, Named Graphs and Blank Nodes. - You are interested in **re-decentralizing the web** or want want to work with tech that improves data ownership and interoperability. -- You like living on the edge (this application is not production ready) ## When _not_ to use this -- If you need **stability**, look further (for now). This is beta sofware and is prone to change. -- You're dealing with **sensitive / private data**. The authorization mechanisms are relatively new and not rigorously tested. -- **Complex query requirements**. Check out NEO4j, Apache Jena or maybe TerminusDB. +- High-throughput **numerical data / numerical analysis**. Atomic-Server does not have aggregate queries. +- If you need **high stability**, look further (for now). This is beta sofware and can change. +- You're dealing with **very sensitive / private data**. The built-in authorization mechanisms are relatively new and not rigorously tested. The database itself is not encrypted. +- **Complex query requirements**. We have queries with filters and features for path traversal, but it may fall short. Check out NEO4j, Apache Jena or maybe TerminusDB. ## Installation & getting started @@ -187,15 +189,17 @@ ATOMIC_SERVER_URL=https://example.com ### Using `systemd` to run Atomic-Server as a service -In Unix operating systems, you can use `systemd` to manage a +In Linux operating systems, you can use `systemd` to manage running processes. +You can configure it to restart automatically, and collect logs with `journalctl`. Create a service: ```sh -vim /etc/systemd/system/atomic.service +nano /etc/systemd/system/atomic.service ``` -Paste this: +Add this to its contents, make changes if needed: + ``` [Unit] Description=Atomic-Server @@ -216,15 +220,11 @@ WantedBy=multi-user.target ``` ```sh -# start service +# start / status / restart commands: systemctl start atomic -# check status systemctl status atomic -# restart systemctl restart atomic -# logs -journalctl -u atomic.service -# logs, since one hour, follow +# show recent logs, follow them on screen journalctl -u atomic.service --since "1 hour ago" -f ``` ## Usage From 4ae12ab59efcdaaaecb62967b23678df9d74729e Mon Sep 17 00:00:00 2001 From: Joep Meindertsma Date: Fri, 10 Feb 2023 10:24:24 +0100 Subject: [PATCH 7/8] Fix doc URLs --- README.md | 2 +- cli/README.md | 10 +++++----- lib/README.md | 2 +- server/README.md | 3 ++- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index cbcc58c6c..a4e3ef49c 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![Discord chat][discord-badge]][discord-url] [![MIT licensed](https://img.shields.io/badge/license-MIT-blue.svg)](./LICENSE) -[![github](https://img.shields.io/github/stars/joepio/atomic?style=social)](https://github.com/atomicdata-dev/atomic-data-browser) +[![github](https://img.shields.io/github/stars/atomicdata-dev/atomic-data-browser?style=social)](https://github.com/atomicdata-dev/atomic-data-browser) **Create, share, fetch and model [Atomic Data](https://docs.atomicdata.dev)! This repo consists of three components: A library, a server and a CLI.** diff --git a/cli/README.md b/cli/README.md index 26400e26e..b6985271a 100644 --- a/cli/README.md +++ b/cli/README.md @@ -3,7 +3,7 @@ [![crates.io](https://img.shields.io/crates/v/atomic-cli)](https://crates.io/crates/atomic-cli) [![Discord chat](https://img.shields.io/discord/723588174747533393.svg?logo=discord)](https://discord.gg/a72Rv2P) [![MIT licensed](https://img.shields.io/badge/license-MIT-blue.svg)](./LICENSE) -[![github](https://img.shields.io/github/stars/joepio/atomic?style=social)](https://github.com/joepio/aget_basetomic) +[![github](https://img.shields.io/github/stars/atomicdata-dev/atomic-data-rust?style=social)](https://github.com/joepio/aget_basetomic) _Status: Beta. [Breaking changes](../CHANGELOG.md) are expected until 1.0._ @@ -39,18 +39,18 @@ Visit https://atomicdata.dev for more info ## Installation -Install [Cargo](https://doc.rust-lang.org/cargo/getting-started/installation.html) to build from source. +You can install `atomic-cli: in multiple ways: -Install using crates.io: +### Using [Cargo](https://doc.rust-lang.org/cargo/getting-started/installation.html) ```sh cargo install atomic-cli ``` -Or build from this repo: +### Build from source ```sh -git clone git@github.com:joepio/atomic.git +git clone git@github.com:atomicdata-dev/atomic-data-rust.git cd atomic/cli # Install atomic to path cargo install --path ./ diff --git a/lib/README.md b/lib/README.md index f40e16e3c..31b23f79f 100644 --- a/lib/README.md +++ b/lib/README.md @@ -4,7 +4,7 @@ [![Released API docs](https://docs.rs/atomic_lib/badge.svg)](https://docs.rs/atomic_lib) [![Discord chat](https://img.shields.io/discord/723588174747533393.svg?logo=discord)](https://discord.gg/a72Rv2P) [![MIT licensed](https://img.shields.io/badge/license-MIT-blue.svg)](./LICENSE) -[![github](https://img.shields.io/github/stars/joepio/atomic?style=social)](https://github.com/joepipo/atomic) +[![github](https://img.shields.io/github/stars/atomicdata-dev/atomic-data-rust?style=social)](https://github.com/joepipo/atomic) _Status: Beta. [Breaking changes](../CHANGELOG.md) are expected until 1.0._ diff --git a/server/README.md b/server/README.md index 283e70c4e..3da0ed494 100644 --- a/server/README.md +++ b/server/README.md @@ -3,7 +3,8 @@ [![crates.io](https://img.shields.io/crates/v/atomic-server)](https://crates.io/crates/atomic-server) [![Discord chat](https://img.shields.io/discord/723588174747533393.svg?logo=discord)](https://discord.gg/a72Rv2P) [![MIT licensed](https://img.shields.io/badge/license-MIT-blue.svg)](./LICENSE) -[![github](https://img.shields.io/github/stars/joepio/atomic?style=social)](https://github.com/atomicdata-dev/atomic-data-browser) +[![github](https://img.shields.io/github/stars/atomicdata-dev/atomic-data-rust +?style=social)](https://github.com/atomicdata-dev/atomic-data-browser) _Status: alpha. [Breaking changes](../CHANGELOG.md) are expected until 1.0._ From 6fc885e5486995bb12710275fbd0af3cf804f5bc Mon Sep 17 00:00:00 2001 From: Joep Meindertsma Date: Fri, 10 Feb 2023 13:36:42 +0100 Subject: [PATCH 8/8] Add `https-dns` option #192 --- CHANGELOG.md | 2 +- lib/src/client.rs | 4 +- server/src/config.rs | 24 ++++-- server/src/https.rs | 185 +++++++++++++++++++++++++++---------------- server/src/serve.rs | 4 +- 5 files changed, 139 insertions(+), 80 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9687a0939..1d0181d6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,7 @@ See [STATUS.md](server/STATUS.md) to learn more about which features will remain - Meta tags server side #577 - Include JSON-AD in initial response, speed up first render #511 - Remove feature to index external RDF files and search them #579 -- Replace `acme_lib` with `instant-acme`, drop OpenSSL dependency #192 +- Replace `acme_lib` with `instant-acme`, drop OpenSSL dependency, add DNS verification for TLS option with `--https-dns` #192 - Add staging environment #588 - Add systemd instructions to readme #271 diff --git a/lib/src/client.rs b/lib/src/client.rs index 33ff01cf6..455f27dce 100644 --- a/lib/src/client.rs +++ b/lib/src/client.rs @@ -13,7 +13,7 @@ use crate::{ /// Checks the datatypes for the Values. /// Ignores all atoms where the subject is different. /// WARNING: Calls store methods, and is called by store methods, might get stuck in a loop! -#[tracing::instrument(skip(store))] +#[tracing::instrument(skip(store), level = "info")] pub fn fetch_resource( subject: &str, store: &impl Storelike, @@ -63,7 +63,7 @@ pub fn fetch_body(url: &str, content_type: &str, for_agent: Option) -> At .get(url) .set("Accept", content_type) .call() - .map_err(|e| format!("Error when fetching {} : {}", url, e))?; + .map_err(|e| format!("Error when server tried fetching {} : {}", url, e))?; let status = resp.status(); let body = resp .into_string() diff --git a/server/src/config.rs b/server/src/config.rs index 7e761eefd..369d2f073 100644 --- a/server/src/config.rs +++ b/server/src/config.rs @@ -31,17 +31,18 @@ pub struct Opts { #[clap(long, default_value = "localhost", env = "ATOMIC_DOMAIN")] pub domain: String, - /// The contact mail address for Let's Encrypt HTTPS setup - #[clap(long, env = "ATOMIC_EMAIL")] - pub email: Option, - // 9.883 is decimal for the `⚛` character. /// The port where the HTTP app is available. Set to 80 if you want this to be available on the network. #[clap(short, long, default_value = "9883", env = "ATOMIC_PORT")] pub port: u32, - /// The port where the HTTPS app is available. Sert to 443 if you want this to be available on the network. - #[clap(long, default_value = "9884", env = "ATOMIC_PORT_HTTPS")] + /// The port where the HTTPS app is available. Set to 443 if you want this to be available on the network. + #[clap( + long, + default_value = "9884", + env = "ATOMIC_PORT_HTTPS", + requires = "https" + )] pub port_https: u32, /// The IP address of the server. Set to :: if you want this to be available to other devices on your network. @@ -49,10 +50,18 @@ pub struct Opts { pub ip: IpAddr, /// Use HTTPS instead of HTTP. - /// Will get certificates from LetsEncrypt. + /// Will get certificates from LetsEncrypt fully automated. #[clap(long, env = "ATOMIC_HTTPS")] pub https: bool, + /// Initializes DNS-01 challenge for LetsEncrypt. Use this if you want to use subdomains. + #[clap(long, env = "ATOMIC_HTTPS_DNS", requires = "https")] + pub https_dns: bool, + + /// The contact mail address for Let's Encrypt HTTPS setup + #[clap(long, env = "ATOMIC_EMAIL")] + pub email: Option, + /// Endpoint where the front-end assets are hosted #[clap(long, default_value = "/app_assets", env = "ATOMIC_ASSET_URL")] pub asset_url: String, @@ -86,6 +95,7 @@ pub struct Opts { pub log_level: LogLevel, /// How you want to trace what's going on with the server. Useful for monitoring performance and errors in production. + /// Combine with `log_level` to get more or less data (`trace` is the most verbose) #[clap(arg_enum, long, env = "ATOMIC_TRACING", default_value = "stdout")] pub trace: Tracing, } diff --git a/server/src/https.rs b/server/src/https.rs index c9eb7a978..6d73de39a 100644 --- a/server/src/https.rs +++ b/server/src/https.rs @@ -1,4 +1,8 @@ -//! Everything required for getting HTTPS config from storage. +//! Everything required for setting up HTTPS / TLS. +//! Instantiate a server for HTTP-01 check with letsencrypt, +//! checks if certificates are not outdated, +//! persists files on disk. + use std::{ fs::{self, File}, io::BufReader, @@ -6,7 +10,7 @@ use std::{ }; use crate::errors::AtomicServerResult; -// RUSTLS +/// Create RUSTLS server config from certificates in config dir pub fn get_https_config( config: &crate::config::Config, ) -> AtomicServerResult { @@ -35,13 +39,12 @@ pub fn get_https_config( } pub fn certs_created_at_path(config: &crate::config::Config) -> PathBuf { - // ~/.config/atomic/https let mut path = config .cert_path .parent() .unwrap_or_else(|| { panic!( - "Cannot open parent dit of HTTPS certs {:?}", + "Cannot open parent dir of HTTPS certs {:?}", config.cert_path ) }) @@ -51,7 +54,7 @@ pub fn certs_created_at_path(config: &crate::config::Config) -> PathBuf { } /// Adds a file to the .https folder to indicate age of certificates -pub fn set_certs_created_at_file(config: &crate::config::Config) { +fn set_certs_created_at_file(config: &crate::config::Config) { let now_string = chrono::Utc::now(); let path = certs_created_at_path(config); fs::write(&path, now_string.to_string()) @@ -60,40 +63,44 @@ pub fn set_certs_created_at_file(config: &crate::config::Config) { /// Checks if the certificates need to be renewed. /// Will be true if there are no certs yet. -pub fn should_renew_certs_check(config: &crate::config::Config) -> bool { +pub fn should_renew_certs_check(config: &crate::config::Config) -> AtomicServerResult { if std::fs::File::open(&config.cert_path).is_err() { - return true; + return Ok(true); } let path = certs_created_at_path(config); let created_at = std::fs::read_to_string(&path) - .unwrap_or_else(|_| panic!("Unable to read {:?}", &path)) + .map_err(|_| format!("Unable to read {:?}", &path))? .parse::>() - .unwrap_or_else(|_| panic!("failed to parse {:?}", &path)); + .map_err(|_| format!("failed to parse {:?}", &path))?; let certs_age: chrono::Duration = chrono::Utc::now() - created_at; // Let's Encrypt certificates are valid for three months, but I think renewing earlier provides a better UX. let expired = certs_age > chrono::Duration::weeks(4); if expired { - tracing::warn!("HTTPS Certificates expired, requesting new ones...") + warn!("HTTPS Certificates expired, requesting new ones...") // This is where I might need to remove the `.https/` folder, but it seems like it's not necessary }; - expired + Ok(expired) } -use actix_web::{App, HttpServer}; -use instant_acme::OrderStatus; -use tracing::info; +use actix_web::{dev::ServerHandle, App, HttpServer}; +use instant_acme::{KeyAuthorization, OrderStatus}; +use tracing::{info, log::warn}; use std::sync::mpsc; /// Starts an HTTP Actix server for HTTPS certificate initialization -pub async fn cert_init_server(config: &crate::config::Config) -> AtomicServerResult<()> { +async fn cert_init_server( + config: &crate::config::Config, + challenge: &instant_acme::Challenge, + key_auth: &KeyAuthorization, +) -> AtomicServerResult { let address = format!("{}:{}", config.opts.ip, config.opts.port); - tracing::warn!("Server temporarily running in HTTP mode at {}, running Let's Encrypt Certificate initialization...", address); + warn!("Server temporarily running in HTTP mode at {}, running Let's Encrypt Certificate initialization...", address); if config.opts.port != 80 { - tracing::warn!( - "HTTP port is {}, not 80. Should be 80 in most cases during LetsEncrypt setup.", + warn!( + "HTTP port is {}, not 80. Should be 80 in most cases during LetsEncrypt setup. If you've correctly forwarded it, you can ignore this warning.", config.opts.port ); } @@ -102,9 +109,14 @@ pub async fn cert_init_server(config: &crate::config::Config) -> AtomicServerRes well_known_folder.push("well-known"); fs::create_dir_all(&well_known_folder)?; - let (tx, rx) = mpsc::channel(); + let mut challenge_path = well_known_folder.clone(); + challenge_path.push("acme-challenge"); + fs::create_dir_all(&challenge_path)?; + challenge_path.push(&challenge.token); + // let challenge_file_content = format!("{}.{}", challenge.token, key_auth.as_str()); + fs::write(challenge_path, &key_auth.as_str())?; - let address_clone = address.clone(); + let (tx, rx) = mpsc::channel(); std::thread::spawn(move || { actix_web::rt::System::new().block_on(async move { @@ -115,7 +127,7 @@ pub async fn cert_init_server(config: &crate::config::Config) -> AtomicServerRes ) }); - let running_server = init_server.bind(&address_clone)?.run(); + let running_server = init_server.bind(&address)?.run(); tx.send(running_server.handle()) .expect("Error sending handle during HTTPS init."); @@ -128,44 +140,44 @@ pub async fn cert_init_server(config: &crate::config::Config) -> AtomicServerRes .recv() .map_err(|e| format!("Error receiving handle during HTTPS init. {}", e))?; + let well_known_url = format!( + "http://{}/.well-known/acme-challenge/{}", + &config.opts.domain, &challenge.token + ); + + // wait for a few secs + std::thread::sleep(std::time::Duration::from_secs(2)); + info!("Testing availability of {}", &well_known_url); + let agent = ureq::builder() .timeout(std::time::Duration::from_secs(2)) .build(); - - let well_known_url = format!("http://{}/.well-known/", &config.opts.domain); - tracing::info!("Testing availability of {}", &well_known_url); - let resp = agent.get(&well_known_url).call().map_err(|e| { - format!( - "Unable to send request for Let's Encrypt initialization. {}", - e - ) - })?; + let resp = agent + .get(&well_known_url) + // .get("https://docs.certifytheweb.com/docs/http-validation/") + .call() + .map_err(|e| format!("Unable to Test local server. {}", e))?; if resp.status() != 200 { - return Err( - "Server for HTTP initialization not available, returning a non-200 status code".into(), - ); + warn!("Unable to Test local server. Status: {}", resp.status()); } else { - tracing::info!("Server for HTTP initialization running correctly"); + info!("Server for HTTP initialization running correctly"); } - - request_cert(config) - .await - .map_err(|e| format!("Certification init failed: {}", e))?; - tracing::warn!("HTTPS TLS Cert init successful! Stopping HTTP server, starting HTTPS..."); - handle.stop(true).await; - Ok(()) + Ok(handle) } -async fn request_cert(config: &crate::config::Config) -> AtomicServerResult<()> { - let use_wildcard = false; - - fs::create_dir_all(PathBuf::from(&config.https_path))?; +/// Sends a request to LetsEncrypt to create a certificate +pub async fn request_cert(config: &crate::config::Config) -> AtomicServerResult<()> { + let challenge_type = if config.opts.https_dns { + instant_acme::ChallengeType::Dns01 + } else { + instant_acme::ChallengeType::Http01 + }; // Create a new account. This will generate a fresh ECDSA key for you. // Alternatively, restore an account from serialized credentials by // using `Account::from_credentials()`. - let url = if config.opts.development { + let lets_encrypt_url = if config.opts.development { instant_acme::LetsEncrypt::Staging.url() } else { instant_acme::LetsEncrypt::Production.url() @@ -184,7 +196,7 @@ async fn request_cert(config: &crate::config::Config) -> AtomicServerResult<()> terms_of_service_agreed: true, only_return_existing: false, }, - url, + lets_encrypt_url, ) .await .map_err(|e| format!("Failed to create account: {}", e))?; @@ -194,7 +206,8 @@ async fn request_cert(config: &crate::config::Config) -> AtomicServerResult<()> // process multiple orders in parallel for a single account. let mut domain = config.opts.domain.clone(); - if use_wildcard { + if config.opts.https_dns { + // Set a wildcard subdomain. Not possible with Http-01 challenge, only Dns-01. domain = format!("*.{}", domain); } let identifier = instant_acme::Identifier::Dns(domain); @@ -205,13 +218,16 @@ async fn request_cert(config: &crate::config::Config) -> AtomicServerResult<()> .await .unwrap(); - tracing::info!("order state: {:#?}", state); assert!(matches!(state.status, instant_acme::OrderStatus::Pending)); // Pick the desired challenge type and prepare the response. let authorizations = order.authorizations(&state.authorizations).await.unwrap(); let mut challenges = Vec::with_capacity(authorizations.len()); + + // if we have H11p01 challenges, we need to start a server to handle them, and eventually turn that off again + let mut handle: Option = None; + for authz in &authorizations { match authz.status { instant_acme::AuthorizationStatus::Pending => {} @@ -222,18 +238,29 @@ async fn request_cert(config: &crate::config::Config) -> AtomicServerResult<()> let challenge = authz .challenges .iter() - .find(|c| c.r#type == instant_acme::ChallengeType::Dns01) + .find(|c| c.r#type == challenge_type) .ok_or("no Dns01 challenge found")?; let instant_acme::Identifier::Dns(identifier) = &authz.identifier; - println!("Please set the following DNS record then press any key:"); - println!( - "_acme-challenge.{} IN TXT {}", - identifier, - order.key_authorization(challenge).dns_value() - ); - std::io::stdin().read_line(&mut String::new()).unwrap(); + let key_auth = order.key_authorization(challenge); + match challenge_type { + instant_acme::ChallengeType::Http01 => { + handle = Some(cert_init_server(config, challenge, &key_auth).await?); + } + instant_acme::ChallengeType::Dns01 => { + // For DNS challenges, we need the user to set a TXT record. + + println!("Please set the following DNS record then press any key:"); + println!( + "_acme-challenge.{} IN TXT {}", + identifier, + key_auth.dns_value() + ); + std::io::stdin().read_line(&mut String::new()).unwrap(); + } + instant_acme::ChallengeType::TlsAlpn01 => todo!("TLS-ALPN-01 is not supported"), + } challenges.push((identifier, &challenge.url)); } @@ -246,27 +273,30 @@ async fn request_cert(config: &crate::config::Config) -> AtomicServerResult<()> // Exponentially back off until the order becomes ready or invalid. let mut tries = 1u8; let mut delay = std::time::Duration::from_millis(250); + let url = authorizations.get(0).expect("Authorizations is empty"); let state = loop { actix::clock::sleep(delay).await; let state = order.state().await.unwrap(); if let instant_acme::OrderStatus::Ready | instant_acme::OrderStatus::Invalid = state.status { - tracing::info!("order state: {:#?}", state); + info!("order state: {:#?}", state); break state; } delay *= 2; tries += 1; - match tries < 5 { - true => info!(?state, tries, "order is not ready, waiting {delay:?}"), + match tries < 100 { + true => info!("order is not ready, waiting {delay:?}"), false => { - return Err("order is not ready".into()); + return Err( + format!("order is not ready. For details, see the url: {url:?}").into(), + ); } } }; if state.status == OrderStatus::Invalid { - return Err("order is invalid".into()); + return Err(format!("order is invalid, check {url:?}").into()); } let mut names = Vec::with_capacity(challenges.len()); @@ -279,21 +309,40 @@ async fn request_cert(config: &crate::config::Config) -> AtomicServerResult<()> let mut params = rcgen::CertificateParams::new(names.clone()); params.distinguished_name = rcgen::DistinguishedName::new(); - let cert = rcgen::Certificate::from_params(params).unwrap(); + let cert = rcgen::Certificate::from_params(params).map_err(|e| e.to_string())?; let csr = cert.serialize_request_der().map_err(|e| e.to_string())?; // Finalize the order and print certificate chain, private key and account credentials. - let cert_chain_pem = order.finalize(&csr, &state.finalize).await.unwrap(); + let cert_chain_pem = order + .finalize(&csr, &state.finalize) + .await + .map_err(|e| e.to_string())?; info!("certficate chain:\n\n{}", cert_chain_pem,); info!("private key:\n\n{}", cert.serialize_private_key_pem()); info!( "account credentials:\n\n{}", - serde_json::to_string_pretty(&account.credentials()).unwrap() + serde_json::to_string_pretty(&account.credentials()).map_err(|e| e.to_string())? ); - fs::write(&config.cert_path, cert_chain_pem).expect("Unable to write cert file"); - fs::write(&config.key_path, cert.serialize_private_key_pem()) - .expect("Unable to write key file"); + write_certs(config, cert_chain_pem, cert)?; + + if let Some(hnd) = handle { + warn!("HTTPS TLS Cert init successful! Stopping temporary HTTP server, starting HTTPS..."); + hnd.stop(true).await; + } + + Ok(()) +} + +fn write_certs( + config: &crate::config::Config, + cert_chain_pem: String, + cert: rcgen::Certificate, +) -> AtomicServerResult<()> { + info!("Writing TLS certificates to {:?}", config.https_path); + fs::create_dir_all(PathBuf::from(&config.https_path))?; + fs::write(&config.cert_path, cert_chain_pem)?; + fs::write(&config.key_path, cert.serialize_private_key_pem())?; set_certs_created_at_file(config); Ok(()) diff --git a/server/src/serve.rs b/server/src/serve.rs index b6a9f663e..195a19aac 100644 --- a/server/src/serve.rs +++ b/server/src/serve.rs @@ -72,8 +72,8 @@ pub async fn serve(config: crate::config::Config) -> AtomicServerResult<()> { { // If there is no certificate file, or the certs are too old, start HTTPS initialization { - if crate::https::should_renew_certs_check(&config) { - crate::https::cert_init_server(&config).await?; + if crate::https::should_renew_certs_check(&config)? { + crate::https::request_cert(&config).await?; } } let https_config = crate::https::get_https_config(&config)