From 0d9d8fc8fbd36f14a8aded966b27e94c5d143043 Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Thu, 16 Nov 2023 09:36:46 -0300 Subject: [PATCH 1/3] add CoreLightningRestWallet --- .github/workflows/ci.yml | 3 +- .github/workflows/regtest.yml | 16 +- cashu/core/settings.py | 7 + cashu/lightning/__init__.py | 1 + cashu/lightning/corelightningrest.py | 308 +++++++++++++++++++++++++++ cashu/lightning/lndrest.py | 31 +-- cashu/mint/crud.py | 19 +- 7 files changed, 343 insertions(+), 42 deletions(-) create mode 100644 cashu/lightning/corelightningrest.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d4237433..07eb7208 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,8 @@ jobs: matrix: python-version: ["3.10"] poetry-version: ["1.5.1"] - backend-wallet-class: ["LndRestWallet", "LNbitsWallet"] + backend-wallet-class: + ["LndRestWallet", "CoreLightningRestWallet", "LNbitsWallet"] with: python-version: ${{ matrix.python-version }} backend-wallet-class: ${{ matrix.backend-wallet-class }} diff --git a/.github/workflows/regtest.yml b/.github/workflows/regtest.yml index 9eeda033..69c2e932 100644 --- a/.github/workflows/regtest.yml +++ b/.github/workflows/regtest.yml @@ -55,14 +55,14 @@ jobs: MINT_LND_REST_ENDPOINT: https://localhost:8081/ MINT_LND_REST_CERT: ./regtest/data/lnd-3/tls.cert MINT_LND_REST_MACAROON: ./regtest/data/lnd-3/data/chain/bitcoin/regtest/admin.macaroon - # LND_GRPC_ENDPOINT: localhost - # LND_GRPC_PORT: 10009 - # LND_GRPC_CERT: docker/data/lnd-3/tls.cert - # LND_GRPC_MACAROON: docker/data/lnd-3/data/chain/bitcoin/regtest/admin.macaroon - # CORELIGHTNING_RPC: ./docker/data/clightning-1/regtest/lightning-rpc - # CORELIGHTNING_REST_URL: https://localhost:3001 - # CORELIGHTNING_REST_MACAROON: ./docker/data/clightning-2-rest/access.macaroon - # CORELIGHTNING_REST_CERT: ./docker/data/clightning-2-rest/certificate.pem + # LND_GRPC_ENDPOINT: localhost + # LND_GRPC_PORT: 10009 + # LND_GRPC_CERT: ./regtest/data/lnd-3/tls.cert + # LND_GRPC_MACAROON: ./regtest/data/lnd-3/data/chain/bitcoin/regtest/admin.macaroon + # CORELIGHTNING_RPC: ./regtest/data/clightning-1/regtest/lightning-rpc + CORELIGHTNING_REST_URL: https://localhost:3001 + CORELIGHTNING_REST_MACAROON: ./regtest/data/clightning-2-rest/access.macaroon + CORELIGHTNING_REST_CERT: ./regtest/data/clightning-2-rest/certificate.pem run: | sudo chmod -R 777 . make test diff --git a/cashu/core/settings.py b/cashu/core/settings.py index 39922029..274e1098 100644 --- a/cashu/core/settings.py +++ b/cashu/core/settings.py @@ -107,9 +107,16 @@ class LndRestFundingSource(MintSettings): mint_lnd_rest_invoice_macaroon: Optional[str] = Field(default=None) +class CoreLightningRestFundingSource(MintSettings): + mint_corelightning_rest_url: Optional[str] = Field(default=None) + mint_corelightning_rest_macaroon: Optional[str] = Field(default=None) + mint_corelightning_rest_cert: Optional[str] = Field(default=None) + + class Settings( EnvSettings, LndRestFundingSource, + CoreLightningRestFundingSource, MintSettings, MintInformation, WalletSettings, diff --git a/cashu/lightning/__init__.py b/cashu/lightning/__init__.py index 146c3616..8b3fbc61 100644 --- a/cashu/lightning/__init__.py +++ b/cashu/lightning/__init__.py @@ -1,5 +1,6 @@ # type: ignore from ..core.settings import settings +from .corelightningrest import CoreLightningRestWallet # noqa: F401 from .fake import FakeWallet # noqa: F401 from .lnbits import LNbitsWallet # noqa: F401 from .lndrest import LndRestWallet # noqa: F401 diff --git a/cashu/lightning/corelightningrest.py b/cashu/lightning/corelightningrest.py new file mode 100644 index 00000000..28458ff2 --- /dev/null +++ b/cashu/lightning/corelightningrest.py @@ -0,0 +1,308 @@ +import asyncio +import json +import random +from typing import AsyncGenerator, Dict, Optional + +import httpx +from bolt11 import Bolt11Exception +from bolt11.decode import decode +from loguru import logger + +from ..core.settings import settings +from .base import ( + InvoiceResponse, + PaymentResponse, + PaymentStatus, + StatusResponse, + Unsupported, + Wallet, +) +from .macaroon import load_macaroon + + +class CoreLightningRestWallet(Wallet): + def __init__(self): + macaroon = settings.mint_corelightning_rest_macaroon + assert macaroon, "missing cln-rest macaroon" + + self.macaroon = load_macaroon(macaroon) + + url = settings.mint_corelightning_rest_url + if not url: + raise Exception("missing url for corelightning-rest") + if not macaroon: + raise Exception("missing macaroon for corelightning-rest") + + self.url = url[:-1] if url.endswith("/") else url + self.url = ( + f"https://{self.url}" if not self.url.startswith("http") else self.url + ) + self.auth = { + "macaroon": self.macaroon, + "encodingtype": "hex", + "accept": "application/json", + } + + self.cert = settings.mint_corelightning_rest_cert or False + self.client = httpx.AsyncClient(verify=self.cert, headers=self.auth) + self.last_pay_index = 0 + self.statuses = { + "paid": True, + "complete": True, + "failed": False, + "pending": None, + } + + async def cleanup(self): + try: + await self.client.aclose() + except RuntimeError as e: + logger.warning(f"Error closing wallet connection: {e}") + + async def status(self) -> StatusResponse: + r = await self.client.get(f"{self.url}/v1/channel/localremotebal", timeout=5) + r.raise_for_status() + if r.is_error or "error" in r.json(): + try: + data = r.json() + error_message = data["error"] + except Exception: + error_message = r.text + return StatusResponse( + error_message=( + f"Failed to connect to {self.url}, got: '{error_message}...'" + ), + balance_msat=0, + ) + + data = r.json() + if len(data) == 0: + return StatusResponse(error_message="no data", balance_msat=0) + balance_msat = int(data.get("localBalance") * 1000) + return StatusResponse(error_message=None, balance_msat=balance_msat) + + async def create_invoice( + self, + amount: int, + memo: Optional[str] = None, + description_hash: Optional[bytes] = None, + unhashed_description: Optional[bytes] = None, + **kwargs, + ) -> InvoiceResponse: + label = f"lbl{random.random()}" + data: Dict = { + "amount": amount * 1000, + "description": memo, + "label": label, + } + if description_hash and not unhashed_description: + raise Unsupported( + "'description_hash' unsupported by CoreLightningRest, " + "provide 'unhashed_description'" + ) + + if unhashed_description: + data["description"] = unhashed_description.decode("utf-8") + + if kwargs.get("expiry"): + data["expiry"] = kwargs["expiry"] + + if kwargs.get("preimage"): + data["preimage"] = kwargs["preimage"] + + r = await self.client.post( + f"{self.url}/v1/invoice/genInvoice", + data=data, + ) + + if r.is_error or "error" in r.json(): + try: + data = r.json() + error_message = data["error"] + except Exception: + error_message = r.text + + return InvoiceResponse( + ok=False, + checking_id=None, + payment_request=None, + error_message=error_message, + ) + + data = r.json() + assert "payment_hash" in data + assert "bolt11" in data + # NOTE: use payment_hash when corelightning-rest updates and supports it + # return InvoiceResponse(True, data["payment_hash"], data["bolt11"], None) + return InvoiceResponse( + ok=True, + checking_id=label, + payment_request=data["bolt11"], + error_message=None, + ) + + async def pay_invoice(self, bolt11: str, fee_limit_msat: int) -> PaymentResponse: + try: + invoice = decode(bolt11) + except Bolt11Exception as exc: + return PaymentResponse( + ok=False, + checking_id=None, + fee_msat=None, + preimage=None, + error_message=str(exc), + ) + + if not invoice.amount_msat or invoice.amount_msat <= 0: + error_message = "0 amount invoices are not allowed" + return PaymentResponse( + ok=False, + checking_id=None, + fee_msat=None, + preimage=None, + error_message=error_message, + ) + fee_limit_percent = fee_limit_msat / invoice.amount_msat * 100 + r = await self.client.post( + f"{self.url}/v1/pay", + data={ + "invoice": bolt11, + "maxfeepercent": f"{fee_limit_percent:.11}", + "exemptfee": 0, # so fee_limit_percent is applied even on payments + # with fee < 5000 millisatoshi (which is default value of exemptfee) + }, + timeout=None, + ) + + if r.is_error or "error" in r.json(): + try: + data = r.json() + error_message = data["error"] + except Exception: + error_message = r.text + return PaymentResponse( + ok=False, + checking_id=None, + fee_msat=None, + preimage=None, + error_message=error_message, + ) + + data = r.json() + + if data["status"] != "complete": + return PaymentResponse( + ok=False, + checking_id=None, + fee_msat=None, + preimage=None, + error_message="payment failed", + ) + + checking_id = data["payment_hash"] + preimage = data["payment_preimage"] + fee_msat = data["msatoshi_sent"] - data["msatoshi"] + + return PaymentResponse( + ok=self.statuses.get(data["status"]), + checking_id=checking_id, + fee_msat=fee_msat, + preimage=preimage, + error_message=None, + ) + + async def get_invoice_status(self, checking_id: str) -> PaymentStatus: + # get invoice bolt11 from checking_id + # corelightning-rest wants the "label" here.... + # NOTE: We can get rid of all labels and use payment_hash when + # corelightning-rest updates and supports it + r = await self.client.get( + f"{self.url}/v1/invoice/listInvoices", + params={"label": checking_id}, + ) + try: + r.raise_for_status() + data = r.json() + + if r.is_error or "error" in data or data.get("invoices") is None: + raise Exception("error in cln response") + return PaymentStatus(paid=self.statuses.get(data["invoices"][0]["status"])) + except Exception as e: + logger.error(f"Error getting invoice status: {e}") + return PaymentStatus(paid=None) + + async def get_payment_status(self, checking_id: str) -> PaymentStatus: + from ..mint.crud import get_lightning_invoice + from ..mint.startup import ledger + + payment = await get_lightning_invoice(db=ledger.db, payment_hash=checking_id) + if not payment: + raise ValueError(f"Payment with checking_id {checking_id} not found") + r = await self.client.get( + f"{self.url}/v1/pay/listPays", + params={"invoice": payment.bolt11}, + ) + try: + r.raise_for_status() + data = r.json() + + if r.is_error or "error" in data or not data.get("pays"): + raise Exception("error in corelightning-rest response") + + pay = data["pays"][0] + + fee_msat, preimage = None, None + if self.statuses[pay["status"]]: + # cut off "msat" and convert to int + fee_msat = -int(pay["amount_sent_msat"][:-4]) - int( + pay["amount_msat"][:-4] + ) + preimage = pay["preimage"] + + return PaymentStatus( + paid=self.statuses.get(pay["status"]), + fee_msat=fee_msat, + preimage=preimage, + ) + except Exception as e: + logger.error(f"Error getting payment status: {e}") + return PaymentStatus(paid=None) + + async def paid_invoices_stream(self) -> AsyncGenerator[str, None]: + while True: + try: + url = f"{self.url}/v1/invoice/waitAnyInvoice/{self.last_pay_index}" + async with self.client.stream("GET", url, timeout=None) as r: + async for line in r.aiter_lines(): + inv = json.loads(line) + if "error" in inv and "message" in inv["error"]: + logger.error("Error in paid_invoices_stream:", inv) + raise Exception(inv["error"]["message"]) + try: + paid = inv["status"] == "paid" + self.last_pay_index = inv["pay_index"] + if not paid: + continue + except Exception: + continue + logger.trace(f"paid invoice: {inv}") + yield inv["label"] + # NOTE: use payment_hash when corelightning-rest updates + # and supports it + # payment_hash = inv["payment_hash"] + # yield payment_hash + # hack to return payment_hash if the above shouldn't work + # r = await self.client.get( + # f"{self.url}/v1/invoice/listInvoices", + # params={"label": inv["label"]}, + # ) + # paid_invoce = r.json() + # logger.trace(f"paid invoice: {paid_invoce}") + # yield paid_invoce["invoices"][0]["payment_hash"] + + except Exception as exc: + logger.debug( + f"lost connection to corelightning-rest invoices stream: '{exc}', " + "reconnecting..." + ) + await asyncio.sleep(0.02) diff --git a/cashu/lightning/lndrest.py b/cashu/lightning/lndrest.py index 33c44808..f3f6c8bf 100644 --- a/cashu/lightning/lndrest.py +++ b/cashu/lightning/lndrest.py @@ -15,36 +15,7 @@ StatusResponse, Wallet, ) - - -def load_macaroon(macaroon: str) -> str: - """Returns hex version of a macaroon encoded in base64 or the file path. - - :param macaroon: Macaroon encoded in base64 or file path. - :type macaroon: str - :return: Hex version of macaroon. - :rtype: str - """ - - # if the macaroon is a file path, load it and return hex version - if macaroon.split(".")[-1] == "macaroon": - with open(macaroon, "rb") as f: - macaroon_bytes = f.read() - return macaroon_bytes.hex() - else: - # if macaroon is a provided string - # check if it is hex, if so, return - try: - bytes.fromhex(macaroon) - return macaroon - except ValueError: - pass - # convert the bas64 macaroon to hex - try: - macaroon = base64.b64decode(macaroon).hex() - except Exception: - pass - return macaroon +from .macaroon import load_macaroon class LndRestWallet(Wallet): diff --git a/cashu/mint/crud.py b/cashu/mint/crud.py index bbdf41b0..5a189710 100644 --- a/cashu/mint/crud.py +++ b/cashu/mint/crud.py @@ -301,15 +301,28 @@ async def store_lightning_invoice( async def get_lightning_invoice( db: Database, - id: str, + *, + id: Optional[str] = None, + payment_hash: Optional[str] = None, conn: Optional[Connection] = None, ): + clauses = [] + values: List[Any] = [] + if id: + clauses.append("id = ?") + values.append(id) + if payment_hash: + clauses.append("payment_hash = ?") + values.append(payment_hash) + where = "" + if clauses: + where = f"WHERE {' AND '.join(clauses)}" row = await (conn or db).fetchone( f""" SELECT * from {table_with_schema(db, 'invoices')} - WHERE id = ? + {where} """, - (id,), + tuple(values), ) row_dict = dict(row) return Invoice(**row_dict) if row_dict else None From 6ced05144a845b2243ded57639055a3795d99e1b Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Thu, 16 Nov 2023 09:39:15 -0300 Subject: [PATCH 2/3] add macaroon loader --- cashu/lightning/macaroon.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 cashu/lightning/macaroon.py diff --git a/cashu/lightning/macaroon.py b/cashu/lightning/macaroon.py new file mode 100644 index 00000000..75f19fa6 --- /dev/null +++ b/cashu/lightning/macaroon.py @@ -0,0 +1,34 @@ +import base64 + + +def load_macaroon(macaroon: str) -> str: + """Returns hex version of a macaroon encoded in base64 or the file path. + + :param macaroon: Macaroon encoded in base64 or file path. + :type macaroon: str + :return: Hex version of macaroon. + :rtype: str + """ + + # if the macaroon is a file path, load it and return hex version + if macaroon.split(".")[-1] == "macaroon": + try: + with open(macaroon, "rb") as f: + macaroon_bytes = f.read() + return macaroon_bytes.hex() + except FileNotFoundError: + raise Exception(f"Macaroon file not found: {macaroon}") + else: + # if macaroon is a provided string + # check if it is hex, if so, return + try: + bytes.fromhex(macaroon) + return macaroon + except ValueError: + pass + # convert the bas64 macaroon to hex + try: + macaroon = base64.b64decode(macaroon).hex() + except Exception: + pass + return macaroon From f9c4c06c65037c3bfe0277ce60f15c2cca6bbe5d Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Thu, 16 Nov 2023 09:44:45 -0300 Subject: [PATCH 3/3] add correct config --- .env.example | 5 +++++ .github/workflows/regtest.yml | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.env.example b/.env.example index 55e0d1ae..d3b11cc9 100644 --- a/.env.example +++ b/.env.example @@ -56,6 +56,11 @@ MINT_LND_REST_ENDPOINT=https://127.0.0.1:8086 MINT_LND_REST_CERT="/home/lnd/.lnd/tls.cert" MINT_LND_REST_MACAROON="/home/lnd/.lnd/data/chain/bitcoin/regtest/admin.macaroon" +# CoreLightningRestWallet +MINT_CORELIGHTNING_REST_URL=https://localhost:3001 +MINT_CORELIGHTNING_REST_MACAROON="./clightning-rest/access.macaroon" +MINT_CORELIGHTNING_REST_CERT="./clightning-2-rest/certificate.pem" + # fee to reserve in percent of the amount LIGHTNING_FEE_PERCENT=1.0 # minimum fee to reserve diff --git a/.github/workflows/regtest.yml b/.github/workflows/regtest.yml index 69c2e932..48653736 100644 --- a/.github/workflows/regtest.yml +++ b/.github/workflows/regtest.yml @@ -60,9 +60,9 @@ jobs: # LND_GRPC_CERT: ./regtest/data/lnd-3/tls.cert # LND_GRPC_MACAROON: ./regtest/data/lnd-3/data/chain/bitcoin/regtest/admin.macaroon # CORELIGHTNING_RPC: ./regtest/data/clightning-1/regtest/lightning-rpc - CORELIGHTNING_REST_URL: https://localhost:3001 - CORELIGHTNING_REST_MACAROON: ./regtest/data/clightning-2-rest/access.macaroon - CORELIGHTNING_REST_CERT: ./regtest/data/clightning-2-rest/certificate.pem + MINT_CORELIGHTNING_REST_URL: https://localhost:3001 + MINT_CORELIGHTNING_REST_MACAROON: ./regtest/data/clightning-2-rest/access.macaroon + MINT_CORELIGHTNING_REST_CERT: ./regtest/data/clightning-2-rest/certificate.pem run: | sudo chmod -R 777 . make test