diff --git a/.env.example b/.env.example index 010340bb..e1b56bfc 100644 --- a/.env.example +++ b/.env.example @@ -73,6 +73,8 @@ MINT_LND_TLS_CERT_BASE64="base64 encoded tls cert" #MINT_BTC_ONCHAIN_BACKEND_MAX_AMOUNT=1000000 #MINT_BTC_ONCHAIN_BACKEND_MIN_CONFIRMATIONS=1 +# (optional) enable tracing with jaeger +#MINT_TRACING_JAEGER_ENDPOINT="127.0.0.1:6831" ### environment variables for the fedimint-cli CLI_FEDIMINT_CONNECTION="fed...." \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 3552fb2a..7510aab1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -594,6 +594,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "176dc175b78f56c0f321911d9c8eb2b77a78a4860b9c19db83835fea1a46649b" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-queue" version = "0.3.11" @@ -1441,6 +1450,12 @@ dependencies = [ "serde", ] +[[package]] +name = "integer-encoding" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" + [[package]] name = "ipnet" version = "2.9.0" @@ -1779,6 +1794,8 @@ dependencies = [ "lightning-invoice 0.26.0", "mockall", "moksha-core", + "opentelemetry", + "opentelemetry-jaeger", "reqwest", "secp256k1 0.28.2", "serde", @@ -1793,6 +1810,7 @@ dependencies = [ "tower-http", "tower-service", "tracing", + "tracing-opentelemetry", "tracing-subscriber", "url", "utoipa", @@ -2012,12 +2030,101 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "opentelemetry" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9591d937bc0e6d2feb6f71a559540ab300ea49955229c347a517a28d27784c54" +dependencies = [ + "opentelemetry_api", + "opentelemetry_sdk", +] + +[[package]] +name = "opentelemetry-jaeger" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "876958ba9084f390f913fcf04ddf7bbbb822898867bb0a51cc28f2b9e5c1b515" +dependencies = [ + "async-trait", + "futures-core", + "futures-util", + "opentelemetry", + "opentelemetry-semantic-conventions", + "thrift", +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73c9f9340ad135068800e7f1b24e9e09ed9e7143f5bf8518ded3d3ec69789269" +dependencies = [ + "opentelemetry", +] + +[[package]] +name = "opentelemetry_api" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a81f725323db1b1206ca3da8bb19874bbd3f57c3bcd59471bfb04525b265b9b" +dependencies = [ + "futures-channel", + "futures-util", + "indexmap 1.9.3", + "js-sys", + "once_cell", + "pin-project-lite", + "thiserror", + "urlencoding", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa8e705a0612d48139799fcbaba0d4a90f06277153e43dd2bdc16c6f0edd8026" +dependencies = [ + "async-trait", + "crossbeam-channel", + "futures-channel", + "futures-executor", + "futures-util", + "once_cell", + "opentelemetry_api", + "ordered-float 3.9.2", + "percent-encoding", + "rand", + "regex", + "thiserror", + "tokio", + "tokio-stream", +] + [[package]] name = "option-ext" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "ordered-float" +version = "3.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1e1c390732d15f1d48471625cd92d154e66db2c56645e29a9cd26f4699f72dc" +dependencies = [ + "num-traits", +] + [[package]] name = "overload" version = "0.1.1" @@ -3371,6 +3478,28 @@ dependencies = [ "once_cell", ] +[[package]] +name = "threadpool" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" +dependencies = [ + "num_cpus", +] + +[[package]] +name = "thrift" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" +dependencies = [ + "byteorder", + "integer-encoding", + "log", + "ordered-float 2.10.1", + "threadpool", +] + [[package]] name = "time" version = "0.3.34" @@ -3705,6 +3834,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "tracing-log" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f751112709b4e791d8ce53e32c4ed2d353565a795ce84da2285393f41557bdf2" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + [[package]] name = "tracing-log" version = "0.2.0" @@ -3716,6 +3856,30 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-opentelemetry" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc09e402904a5261e42cf27aea09ccb7d5318c6717a9eec3d8e2e65c56b18f19" +dependencies = [ + "once_cell", + "opentelemetry", + "tracing", + "tracing-core", + "tracing-log 0.1.4", + "tracing-subscriber", +] + +[[package]] +name = "tracing-serde" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6b213177105856957181934e4920de57730fc69bf42c37ee5bb664d406d9e1" +dependencies = [ + "serde", + "tracing-core", +] + [[package]] name = "tracing-subscriber" version = "0.3.18" @@ -3726,12 +3890,15 @@ dependencies = [ "nu-ansi-term", "once_cell", "regex", + "serde", + "serde_json", "sharded-slab", "smallvec", "thread_local", "tracing", "tracing-core", - "tracing-log", + "tracing-log 0.2.0", + "tracing-serde", ] [[package]] diff --git a/docker-compose.yml b/docker-compose.yml index b0eb3994..b860353f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,7 @@ version: "3" +#docker run -d -p6831:6831/udp -p6832:6832/udp -p16686:16686 -p14268:14268 jaegertracing/all-in-one:latest + services: database: image: "postgres:16.2" @@ -9,6 +11,18 @@ services: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_DB: moksha-mint + jaeger: + image: jaegertracing/all-in-one:latest + environment: + - COLLECTOR_OTLP_ENABLED=true + ports: + - 16686:16686 + - 4317:4317 + - 4318:4318 + - 14268:14268 + - 6831:6831/udp + - 6832:6832/udp + restart: always app: image: "docker.io/ngutech21/moksha-mint:latest" #image: "moksha-mint:latest" # for local testing diff --git a/moksha-mint/Cargo.toml b/moksha-mint/Cargo.toml index 7f0d8918..9f10c351 100644 --- a/moksha-mint/Cargo.toml +++ b/moksha-mint/Cargo.toml @@ -26,10 +26,9 @@ hyper = "0.14.28" serde = { version = "1.0.193", features = ["derive"] } serde_json = "1.0.108" tokio = { version = "1.36.0", features = ["full"] } -tower-http = { version = "0.5.0", features = ["trace", "cors", "fs", "set-header"] } +tower-http = { version = "0.5.0", features = ["cors", "fs", "set-header"] } tower-service = { version = "0.3.2" } -tracing = "0.1.40" -tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } + secp256k1 = { version = "0.28.1", default-features = false, features = ["rand", "serde"] } thiserror = "1.0.50" moksha-core = { path = "../moksha-core", version = "0.2.1" } @@ -49,6 +48,12 @@ bitcoin_hashes = "0.13.0" tonic = { version = "0.8", features = ["transport", "tls"] } serde_yaml = "0.9.31" +tracing = "0.1.40" +tracing-subscriber = { version = "0.3.18", features = ["env-filter", "json"] } +opentelemetry-jaeger = { version = "0.19.0" } +opentelemetry = { version = "0.20.0", features = ["trace", "rt-tokio"] } +tracing-opentelemetry = "0.20.0" + [dev-dependencies] tempfile = "3.8.1" tower = { version = "0.4.13", features = ["util"] } diff --git a/moksha-mint/src/bin/moksha-mint.rs b/moksha-mint/src/bin/moksha-mint.rs index ebc3e21e..f72a02c1 100644 --- a/moksha-mint/src/bin/moksha-mint.rs +++ b/moksha-mint/src/bin/moksha-mint.rs @@ -26,6 +26,7 @@ pub async fn main() -> anyhow::Result<()> { database, btconchain_backend, lightning_backend, + tracing, } = MintConfig::read_config_with_defaults(); let mint = MintBuilder::new() @@ -37,6 +38,7 @@ pub async fn main() -> anyhow::Result<()> { .with_lightning(lightning_backend.expect("lightning not set")) .with_btc_onchain(btconchain_backend) .with_fee(Some(lightning_fee)) + .with_tracing(tracing) .build() .await; diff --git a/moksha-mint/src/config.rs b/moksha-mint/src/config.rs index e973021f..294282d5 100644 --- a/moksha-mint/src/config.rs +++ b/moksha-mint/src/config.rs @@ -30,6 +30,9 @@ pub struct Opts { #[clap(long, env = "MINT_BTC_ONCHAIN_BACKEND")] pub btconchain_backend: Option, + + #[clap(flatten)] + pub tracing: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -56,6 +59,12 @@ impl FromStr for LightningTypeVariant { } } +#[derive(Debug, Clone, Default, Parser)] +pub struct TracingConfig { + #[clap(long, env = "MINT_TRACING_JAEGER_ENDPOINT")] + pub jaeger_endpoint: Option, +} + #[derive(Debug, Clone, Default)] pub struct MintConfig { pub privatekey: String, @@ -66,6 +75,7 @@ pub struct MintConfig { pub database: DatabaseConfig, pub btconchain_backend: Option, pub lightning_backend: Option, + pub tracing: Option, } impl From<(Opts, LightningType, Option)> for MintConfig { @@ -79,6 +89,7 @@ impl From<(Opts, LightningType, Option)> for MintConfig { database: opts.database, btconchain_backend: btc, lightning_backend: Some(ln), + tracing: opts.tracing, } } } @@ -121,6 +132,7 @@ impl MintConfig { database: DatabaseConfig, btconchain_backend: Option, lightning_backend: Option, + tracing: Option, ) -> Self { Self { privatekey: private_key, @@ -131,6 +143,7 @@ impl MintConfig { database, btconchain_backend, lightning_backend, + tracing, } } } diff --git a/moksha-mint/src/database/postgres.rs b/moksha-mint/src/database/postgres.rs index d76e8bf0..c47dd528 100644 --- a/moksha-mint/src/database/postgres.rs +++ b/moksha-mint/src/database/postgres.rs @@ -8,6 +8,7 @@ use moksha_core::{ }; use sqlx::postgres::PgPoolOptions; +use tracing::instrument; use uuid::Uuid; use crate::{config::DatabaseConfig, error::MokshaMintError, model::Invoice}; @@ -84,6 +85,7 @@ impl Database for PostgresDB { Ok(()) } + #[instrument(level = "debug", skip(self))] async fn get_pending_invoice(&self, key: String) -> Result { let invoice: Invoice = sqlx::query!( "SELECT amount, payment_request FROM pending_invoices WHERE key = $1", @@ -99,6 +101,7 @@ impl Database for PostgresDB { Ok(invoice) } + #[instrument(level = "debug", skip(self))] async fn add_pending_invoice( &self, key: String, @@ -123,6 +126,7 @@ impl Database for PostgresDB { Ok(()) } + #[instrument(level = "debug", skip(self), err)] async fn get_bolt11_mint_quote(&self, id: &Uuid) -> Result { let quote: Bolt11MintQuote = sqlx::query!( "SELECT id, payment_request, expiry, paid FROM bolt11_mint_quotes WHERE id = $1", @@ -139,6 +143,7 @@ impl Database for PostgresDB { Ok(quote) } + #[instrument(level = "debug", skip(self), err)] async fn add_bolt11_mint_quote(&self, quote: &Bolt11MintQuote) -> Result<(), MokshaMintError> { sqlx::query!( "INSERT INTO bolt11_mint_quotes (id, payment_request, expiry, paid) VALUES ($1, $2, $3, $4)", @@ -152,6 +157,7 @@ impl Database for PostgresDB { Ok(()) } + #[instrument(level = "debug", skip(self), err)] async fn update_bolt11_mint_quote( &self, quote: &Bolt11MintQuote, @@ -166,6 +172,7 @@ impl Database for PostgresDB { Ok(()) } + #[instrument(level = "debug", skip(self), err)] async fn delete_bolt11_mint_quote( &self, quote: &Bolt11MintQuote, diff --git a/moksha-mint/src/lightning/lnd.rs b/moksha-mint/src/lightning/lnd.rs index 8238b9ed..b090a9d7 100644 --- a/moksha-mint/src/lightning/lnd.rs +++ b/moksha-mint/src/lightning/lnd.rs @@ -14,7 +14,7 @@ use clap::Parser; use fedimint_tonic_lnd::Client; use serde::{Deserialize, Serialize}; use tokio::sync::{MappedMutexGuard, Mutex, MutexGuard}; -use tracing::info; +use tracing::{debug, instrument}; use url::Url; use super::Lightning; @@ -78,6 +78,7 @@ impl LndLightning { #[allow(implied_bounds_entailment)] #[async_trait] impl Lightning for LndLightning { + #[instrument(skip(self), err)] async fn is_invoice_paid(&self, payment_request: String) -> Result { let invoice = self.decode_invoice(payment_request).await?; let payment_hash = invoice.payment_hash(); @@ -96,6 +97,7 @@ impl Lightning for LndLightning { Ok(invoice.state == fedimint_tonic_lnd::lnrpc::invoice::InvoiceState::Settled as i32) } + #[instrument(skip(self), err)] async fn create_invoice(&self, amount: u64) -> Result { let invoice_request = fedimint_tonic_lnd::lnrpc::Invoice { value: amount as i64, @@ -115,6 +117,7 @@ impl Lightning for LndLightning { }) } + #[instrument(skip(self), err)] async fn pay_invoice( &self, payment_request: String, @@ -134,7 +137,7 @@ impl Lightning for LndLightning { .payment_route .map_or(0, |route| route.total_fees_msat / 1_000) as u64; - info!("lnd total_fees: {}", total_fees); + debug!("lnd total_fees: {}", total_fees); Ok(PayInvoiceResult { payment_hash: hex::encode(payment_response.payment_hash), diff --git a/moksha-mint/src/mint.rs b/moksha-mint/src/mint.rs index 515fbf46..6e02cfff 100644 --- a/moksha-mint/src/mint.rs +++ b/moksha-mint/src/mint.rs @@ -8,12 +8,13 @@ use moksha_core::{ primitives::{OnchainMeltQuote, PaymentMethod}, proof::Proofs, }; +use tracing::instrument; use crate::{ btconchain::{lnd::LndBtcOnchain, BtcOnchain}, config::{ BtcOnchainConfig, BtcOnchainType, BuildParams, DatabaseConfig, LightningFeeConfig, - MintConfig, MintInfoConfig, ServerConfig, + MintConfig, MintInfoConfig, ServerConfig, TracingConfig, }, database::Database, error::MokshaMintError, @@ -111,6 +112,7 @@ impl Mint { Ok((pr, key)) } + #[instrument(level = "debug", skip(self))] pub async fn mint_tokens( &self, payment_method: PaymentMethod, @@ -277,6 +279,7 @@ pub struct MintBuilder { mint_info_settings: Option, server_config: Option, btc_onchain_config: Option, + tracing_config: Option, } impl MintBuilder { @@ -324,6 +327,11 @@ impl MintBuilder { self } + pub fn with_tracing(mut self, tracing_config: Option) -> Self { + self.tracing_config = tracing_config; + self + } + pub async fn build(self) -> Result { let ln: Arc = match self.lightning_type.clone() { Some(LightningType::Lnbits(lnbits_settings)) => Arc::new(LnbitsLightning::new( @@ -397,6 +405,7 @@ impl MintBuilder { db_config, self.btc_onchain_config, self.lightning_type, + self.tracing_config, ), BuildParams::from_env(), lnd_onchain, diff --git a/moksha-mint/src/routes/default.rs b/moksha-mint/src/routes/default.rs index bb5620c9..b9bff208 100644 --- a/moksha-mint/src/routes/default.rs +++ b/moksha-mint/src/routes/default.rs @@ -12,7 +12,7 @@ use moksha_core::{ PostSwapRequest, PostSwapResponse, }, }; -use tracing::{info, instrument}; +use tracing::{debug, instrument}; use uuid::Uuid; use crate::{ @@ -146,7 +146,7 @@ pub async fn post_mint_quote_bolt11( ("quote_id" = String, Path, description = "quote id"), ) )] -#[instrument(name = "post_mint_bolt11", skip(mint), err)] +#[instrument(name = "post_mint_bolt11", fields(quote_id = %request.quote), skip_all, err)] pub async fn post_mint_bolt11( State(mint): State, Json(request): Json, @@ -196,7 +196,7 @@ pub async fn post_melt_quote_bolt11( crate::error::MokshaMintError::InvalidAmount("invalid invoice".to_owned()) })?; let fee_reserve = mint.fee_reserve(amount) / 1_000; // FIXME check if this is correct - info!("fee_reserve: {}", fee_reserve); + debug!("fee_reserve: {}", fee_reserve); let amount_sat = amount / 1_000; let key = Uuid::new_v4(); @@ -237,7 +237,7 @@ pub async fn post_melt_bolt11( .get_bolt11_melt_quote(&Uuid::from_str(melt_request.quote.as_str())?) .await?; - info!("post_melt_bolt11 fee_reserve: {:#?}", "e); + debug!("post_melt_bolt11 fee_reserve: {:#?}", "e); let (paid, payment_preimage, change) = mint .melt_bolt11( @@ -274,7 +274,7 @@ pub async fn get_mint_quote_bolt11( Path(quote_id): Path, State(mint): State, ) -> Result, MokshaMintError> { - info!("get_quote: {}", quote_id); + debug!("get_quote: {}", quote_id); let quote = mint .db @@ -304,7 +304,7 @@ pub async fn get_melt_quote_bolt11( Path(quote_id): Path, State(mint): State, ) -> Result, MokshaMintError> { - info!("get_melt_quote: {}", quote_id); + debug!("get_melt_quote: {}", quote_id); let quote = mint .db .get_bolt11_melt_quote(&Uuid::from_str(quote_id.as_str())?) diff --git a/moksha-mint/src/server.rs b/moksha-mint/src/server.rs index e0dfb7c8..5828ed2b 100644 --- a/moksha-mint/src/server.rs +++ b/moksha-mint/src/server.rs @@ -19,7 +19,7 @@ use axum::{routing::get, Json}; use moksha_core::keyset::{V1Keyset, V1Keysets}; use moksha_core::proof::Proofs; use moksha_core::proof::{P2SHScript, Proof}; -use tracing_subscriber::EnvFilter; + use utoipa_swagger_ui::SwaggerUi; use crate::mint::Mint; @@ -38,27 +38,44 @@ use moksha_core::primitives::{ use tower_http::services::ServeDir; -use tower_http::{ - cors::{Any, CorsLayer}, - trace::TraceLayer, -}; +use tower_http::cors::{Any, CorsLayer}; use tracing::info; -use tracing_subscriber::prelude::__tracing_subscriber_SubscriberExt; -use tracing_subscriber::util::SubscriberInitExt; - use utoipa::OpenApi; +use tracing_subscriber::{filter::EnvFilter, fmt, layer::SubscriberExt, util::SubscriberInitExt}; + use crate::routes::legacy::{ get_legacy_keys, get_legacy_keysets, get_legacy_mint, post_legacy_check_fees, post_legacy_melt, post_legacy_mint, post_legacy_split, }; pub async fn run_server(mint: Mint) -> anyhow::Result<()> { + let config = mint.config.clone(); + + let jaeger = if config.tracing.is_some() { + let tracer = opentelemetry_jaeger::new_agent_pipeline() + .with_service_name("moksha-mint") + .with_endpoint( + config + .tracing + .unwrap_or_default() + .jaeger_endpoint + .unwrap_or_default(), + ) + .install_simple() + .unwrap(); + + Some(tracing_opentelemetry::layer().with_tracer(tracer)) + } else { + None + }; + tracing_subscriber::registry() - .with(tracing_subscriber::fmt::layer()) + .with(fmt::layer().pretty()) .with(EnvFilter::from_default_env()) - .init(); + .with(jaeger) + .try_init()?; if let Some(ref buildtime) = mint.build_params.build_time { info!("build time: {}", buildtime); @@ -86,6 +103,8 @@ pub async fn run_server(mint: Mint) -> anyhow::Result<()> { info!("btconchain-backend is not configured"); } + info!("tracing jaeger-endpoint: {:?}", mint.config.tracing); + let listener = tokio::net::TcpListener::bind(&mint.config.server.host_port) .await .unwrap(); @@ -232,8 +251,7 @@ fn app(mint: Mint) -> Router { .nest(&prefix, default_routes) .nest(&prefix, btconchain_routes) .nest("", general_routes) - .with_state(mint) - .layer(TraceLayer::new_for_http()); + .with_state(mint); if let Some(ref serve_wallet_path) = server_config.serve_wallet_path { return router.nest_service(