From 18b9db4ae463bb237ab1027677678e7c5d1f33ff Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Sun, 11 Feb 2024 00:49:19 +0100 Subject: [PATCH 01/45] add websockets for quote updates --- cashu/mint/ledger.py | 19 +++++++--- cashu/mint/router.py | 14 +++++++- poetry.lock | 83 +++++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 1 + 4 files changed, 111 insertions(+), 6 deletions(-) diff --git a/cashu/mint/ledger.py b/cashu/mint/ledger.py index 8d0be64b..32b4301a 100644 --- a/cashu/mint/ledger.py +++ b/cashu/mint/ledger.py @@ -7,6 +7,8 @@ import bolt11 from loguru import logger +from cashu.mint.quotes import QuoteQueue + from ..core.base import ( DLEQ, Amount, @@ -63,6 +65,7 @@ class Ledger(LedgerVerification, LedgerSpendingConditions): asyncio.Lock() ) # holds locks for proofs_pending database keysets: Dict[str, MintKeyset] = {} + quote_queue = QuoteQueue() def __init__( self, @@ -355,10 +358,9 @@ async def mint_quote(self, quote_request: PostMintQuoteRequest) -> MintQuote: created_time=int(time.time()), expiry=invoice_obj.expiry or 0, ) - await self.crud.store_mint_quote( - quote=quote, - db=self.db, - ) + await self.crud.store_mint_quote(quote=quote, db=self.db) + await self.quote_queue.submit(quote.quote, quote) + return quote async def get_mint_quote(self, quote_id: str) -> MintQuote: @@ -389,6 +391,7 @@ async def get_mint_quote(self, quote_id: str) -> MintQuote: quote.paid = True quote.paid_time = int(time.time()) await self.crud.update_mint_quote(quote=quote, db=self.db) + await self.quote_queue.submit(quote.quote, quote) return quote @@ -438,6 +441,8 @@ async def mint( logger.trace(f"crud: setting quote {quote_id} as issued") quote.issued = True await self.crud.update_mint_quote(quote=quote, db=self.db) + await self.quote_queue.submit(quote.quote, quote) + del self.locks[quote_id] return promises @@ -509,6 +514,8 @@ async def melt_quote( created_time=int(time.time()), ) await self.crud.store_melt_quote(quote=quote, db=self.db) + await self.quote_queue.submit(quote.quote, quote) + return PostMeltQuoteResponse( quote=quote.quote, amount=quote.amount, @@ -561,6 +568,7 @@ async def get_melt_quote(self, quote_id: str) -> MeltQuote: melt_quote.proof = status.preimage melt_quote.paid_time = int(time.time()) await self.crud.update_melt_quote(quote=melt_quote, db=self.db) + await self.quote_queue.submit(melt_quote.quote, melt_quote) return melt_quote @@ -611,10 +619,12 @@ async def melt_mint_settle_internally(self, melt_quote: MeltQuote) -> MeltQuote: melt_quote.paid = True melt_quote.paid_time = int(time.time()) await self.crud.update_melt_quote(quote=melt_quote, db=self.db) + await self.quote_queue.submit(melt_quote.quote, melt_quote) mint_quote.paid = True mint_quote.paid_time = melt_quote.paid_time await self.crud.update_mint_quote(quote=mint_quote, db=self.db) + await self.quote_queue.submit(mint_quote.quote, mint_quote) return melt_quote @@ -698,6 +708,7 @@ async def melt( melt_quote.paid = True melt_quote.paid_time = int(time.time()) await self.crud.update_melt_quote(quote=melt_quote, db=self.db) + await self.quote_queue.submit(melt_quote.quote, melt_quote) # melt successful, invalidate proofs await self._invalidate_proofs(proofs) diff --git a/cashu/mint/router.py b/cashu/mint/router.py index 02289bcc..f44f6cac 100644 --- a/cashu/mint/router.py +++ b/cashu/mint/router.py @@ -1,6 +1,6 @@ from typing import Any, Dict, List -from fastapi import APIRouter +from fastapi import APIRouter, WebSocket, WebSocketDisconnect from loguru import logger from ..core.base import ( @@ -204,6 +204,18 @@ async def get_mint_quote(quote: str) -> PostMintQuoteResponse: return resp +@router.websocket("/v1/quote/{quote_id}", name="Quote updates") +@router.websocket("/v1/mint/quote/bolt11/{quote_id}", name="Mint quote updates") +@router.websocket("/v1/melt/quote/bolt11/{quote_id}", name="Melt quote updates") +async def websocket_endpoint(websocket: WebSocket, quote_id: str): + await websocket.accept() + try: + async for quote in ledger.quote_queue.watch(quote_id): + await websocket.send_json(quote.dict()) + except WebSocketDisconnect: + pass + + @router.post( "/v1/mint/bolt11", name="Mint tokens with a Lightning payment", diff --git a/poetry.lock b/poetry.lock index ace9c370..270d7937 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1549,6 +1549,87 @@ docs = ["Sphinx (>=6.0)", "sphinx-rtd-theme (>=1.1.0)"] optional = ["python-socks", "wsaccel"] test = ["websockets"] +[[package]] +name = "websockets" +version = "12.0" +description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)" +optional = false +python-versions = ">=3.8" +files = [ + {file = "websockets-12.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d554236b2a2006e0ce16315c16eaa0d628dab009c33b63ea03f41c6107958374"}, + {file = "websockets-12.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2d225bb6886591b1746b17c0573e29804619c8f755b5598d875bb4235ea639be"}, + {file = "websockets-12.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:eb809e816916a3b210bed3c82fb88eaf16e8afcf9c115ebb2bacede1797d2547"}, + {file = "websockets-12.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c588f6abc13f78a67044c6b1273a99e1cf31038ad51815b3b016ce699f0d75c2"}, + {file = "websockets-12.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5aa9348186d79a5f232115ed3fa9020eab66d6c3437d72f9d2c8ac0c6858c558"}, + {file = "websockets-12.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6350b14a40c95ddd53e775dbdbbbc59b124a5c8ecd6fbb09c2e52029f7a9f480"}, + {file = "websockets-12.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:70ec754cc2a769bcd218ed8d7209055667b30860ffecb8633a834dde27d6307c"}, + {file = "websockets-12.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6e96f5ed1b83a8ddb07909b45bd94833b0710f738115751cdaa9da1fb0cb66e8"}, + {file = "websockets-12.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4d87be612cbef86f994178d5186add3d94e9f31cc3cb499a0482b866ec477603"}, + {file = "websockets-12.0-cp310-cp310-win32.whl", hash = "sha256:befe90632d66caaf72e8b2ed4d7f02b348913813c8b0a32fae1cc5fe3730902f"}, + {file = "websockets-12.0-cp310-cp310-win_amd64.whl", hash = "sha256:363f57ca8bc8576195d0540c648aa58ac18cf85b76ad5202b9f976918f4219cf"}, + {file = "websockets-12.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:5d873c7de42dea355d73f170be0f23788cf3fa9f7bed718fd2830eefedce01b4"}, + {file = "websockets-12.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3f61726cae9f65b872502ff3c1496abc93ffbe31b278455c418492016e2afc8f"}, + {file = "websockets-12.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ed2fcf7a07334c77fc8a230755c2209223a7cc44fc27597729b8ef5425aa61a3"}, + {file = "websockets-12.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e332c210b14b57904869ca9f9bf4ca32f5427a03eeb625da9b616c85a3a506c"}, + {file = "websockets-12.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5693ef74233122f8ebab026817b1b37fe25c411ecfca084b29bc7d6efc548f45"}, + {file = "websockets-12.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e9e7db18b4539a29cc5ad8c8b252738a30e2b13f033c2d6e9d0549b45841c04"}, + {file = "websockets-12.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:6e2df67b8014767d0f785baa98393725739287684b9f8d8a1001eb2839031447"}, + {file = "websockets-12.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:bea88d71630c5900690fcb03161ab18f8f244805c59e2e0dc4ffadae0a7ee0ca"}, + {file = "websockets-12.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dff6cdf35e31d1315790149fee351f9e52978130cef6c87c4b6c9b3baf78bc53"}, + {file = "websockets-12.0-cp311-cp311-win32.whl", hash = "sha256:3e3aa8c468af01d70332a382350ee95f6986db479ce7af14d5e81ec52aa2b402"}, + {file = "websockets-12.0-cp311-cp311-win_amd64.whl", hash = "sha256:25eb766c8ad27da0f79420b2af4b85d29914ba0edf69f547cc4f06ca6f1d403b"}, + {file = "websockets-12.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0e6e2711d5a8e6e482cacb927a49a3d432345dfe7dea8ace7b5790df5932e4df"}, + {file = "websockets-12.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:dbcf72a37f0b3316e993e13ecf32f10c0e1259c28ffd0a85cee26e8549595fbc"}, + {file = "websockets-12.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:12743ab88ab2af1d17dd4acb4645677cb7063ef4db93abffbf164218a5d54c6b"}, + {file = "websockets-12.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7b645f491f3c48d3f8a00d1fce07445fab7347fec54a3e65f0725d730d5b99cb"}, + {file = "websockets-12.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9893d1aa45a7f8b3bc4510f6ccf8db8c3b62120917af15e3de247f0780294b92"}, + {file = "websockets-12.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f38a7b376117ef7aff996e737583172bdf535932c9ca021746573bce40165ed"}, + {file = "websockets-12.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:f764ba54e33daf20e167915edc443b6f88956f37fb606449b4a5b10ba42235a5"}, + {file = "websockets-12.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:1e4b3f8ea6a9cfa8be8484c9221ec0257508e3a1ec43c36acdefb2a9c3b00aa2"}, + {file = "websockets-12.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:9fdf06fd06c32205a07e47328ab49c40fc1407cdec801d698a7c41167ea45113"}, + {file = "websockets-12.0-cp312-cp312-win32.whl", hash = "sha256:baa386875b70cbd81798fa9f71be689c1bf484f65fd6fb08d051a0ee4e79924d"}, + {file = "websockets-12.0-cp312-cp312-win_amd64.whl", hash = "sha256:ae0a5da8f35a5be197f328d4727dbcfafa53d1824fac3d96cdd3a642fe09394f"}, + {file = "websockets-12.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5f6ffe2c6598f7f7207eef9a1228b6f5c818f9f4d53ee920aacd35cec8110438"}, + {file = "websockets-12.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9edf3fc590cc2ec20dc9d7a45108b5bbaf21c0d89f9fd3fd1685e223771dc0b2"}, + {file = "websockets-12.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:8572132c7be52632201a35f5e08348137f658e5ffd21f51f94572ca6c05ea81d"}, + {file = "websockets-12.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:604428d1b87edbf02b233e2c207d7d528460fa978f9e391bd8aaf9c8311de137"}, + {file = "websockets-12.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1a9d160fd080c6285e202327aba140fc9a0d910b09e423afff4ae5cbbf1c7205"}, + {file = "websockets-12.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87b4aafed34653e465eb77b7c93ef058516cb5acf3eb21e42f33928616172def"}, + {file = "websockets-12.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b2ee7288b85959797970114deae81ab41b731f19ebcd3bd499ae9ca0e3f1d2c8"}, + {file = "websockets-12.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:7fa3d25e81bfe6a89718e9791128398a50dec6d57faf23770787ff441d851967"}, + {file = "websockets-12.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:a571f035a47212288e3b3519944f6bf4ac7bc7553243e41eac50dd48552b6df7"}, + {file = "websockets-12.0-cp38-cp38-win32.whl", hash = "sha256:3c6cc1360c10c17463aadd29dd3af332d4a1adaa8796f6b0e9f9df1fdb0bad62"}, + {file = "websockets-12.0-cp38-cp38-win_amd64.whl", hash = "sha256:1bf386089178ea69d720f8db6199a0504a406209a0fc23e603b27b300fdd6892"}, + {file = "websockets-12.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ab3d732ad50a4fbd04a4490ef08acd0517b6ae6b77eb967251f4c263011a990d"}, + {file = "websockets-12.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a1d9697f3337a89691e3bd8dc56dea45a6f6d975f92e7d5f773bc715c15dde28"}, + {file = "websockets-12.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1df2fbd2c8a98d38a66f5238484405b8d1d16f929bb7a33ed73e4801222a6f53"}, + {file = "websockets-12.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23509452b3bc38e3a057382c2e941d5ac2e01e251acce7adc74011d7d8de434c"}, + {file = "websockets-12.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e5fc14ec6ea568200ea4ef46545073da81900a2b67b3e666f04adf53ad452ec"}, + {file = "websockets-12.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46e71dbbd12850224243f5d2aeec90f0aaa0f2dde5aeeb8fc8df21e04d99eff9"}, + {file = "websockets-12.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b81f90dcc6c85a9b7f29873beb56c94c85d6f0dac2ea8b60d995bd18bf3e2aae"}, + {file = "websockets-12.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:a02413bc474feda2849c59ed2dfb2cddb4cd3d2f03a2fedec51d6e959d9b608b"}, + {file = "websockets-12.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:bbe6013f9f791944ed31ca08b077e26249309639313fff132bfbf3ba105673b9"}, + {file = "websockets-12.0-cp39-cp39-win32.whl", hash = "sha256:cbe83a6bbdf207ff0541de01e11904827540aa069293696dd528a6640bd6a5f6"}, + {file = "websockets-12.0-cp39-cp39-win_amd64.whl", hash = "sha256:fc4e7fa5414512b481a2483775a8e8be7803a35b30ca805afa4998a84f9fd9e8"}, + {file = "websockets-12.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:248d8e2446e13c1d4326e0a6a4e9629cb13a11195051a73acf414812700badbd"}, + {file = "websockets-12.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f44069528d45a933997a6fef143030d8ca8042f0dfaad753e2906398290e2870"}, + {file = "websockets-12.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c4e37d36f0d19f0a4413d3e18c0d03d0c268ada2061868c1e6f5ab1a6d575077"}, + {file = "websockets-12.0-pp310-pypy310_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d829f975fc2e527a3ef2f9c8f25e553eb7bc779c6665e8e1d52aa22800bb38b"}, + {file = "websockets-12.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:2c71bd45a777433dd9113847af751aae36e448bc6b8c361a566cb043eda6ec30"}, + {file = "websockets-12.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:0bee75f400895aef54157b36ed6d3b308fcab62e5260703add87f44cee9c82a6"}, + {file = "websockets-12.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:423fc1ed29f7512fceb727e2d2aecb952c46aa34895e9ed96071821309951123"}, + {file = "websockets-12.0-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:27a5e9964ef509016759f2ef3f2c1e13f403725a5e6a1775555994966a66e931"}, + {file = "websockets-12.0-pp38-pypy38_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c3181df4583c4d3994d31fb235dc681d2aaad744fbdbf94c4802485ececdecf2"}, + {file = "websockets-12.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:b067cb952ce8bf40115f6c19f478dc71c5e719b7fbaa511359795dfd9d1a6468"}, + {file = "websockets-12.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:00700340c6c7ab788f176d118775202aadea7602c5cc6be6ae127761c16d6b0b"}, + {file = "websockets-12.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e469d01137942849cff40517c97a30a93ae79917752b34029f0ec72df6b46399"}, + {file = "websockets-12.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffefa1374cd508d633646d51a8e9277763a9b78ae71324183693959cf94635a7"}, + {file = "websockets-12.0-pp39-pypy39_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba0cab91b3956dfa9f512147860783a1829a8d905ee218a9837c18f683239611"}, + {file = "websockets-12.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:2cb388a5bfb56df4d9a406783b7f9dbefb888c09b71629351cc6b036e9259370"}, + {file = "websockets-12.0-py3-none-any.whl", hash = "sha256:dc284bbc8d7c78a6c69e0c7325ab46ee5e40bb4d50e494d8131a07ef47500e9e"}, + {file = "websockets-12.0.tar.gz", hash = "sha256:81df9cbcbb6c260de1e007e58c011bfebe2dafc8435107b0537f393dd38c8b1b"}, +] + [[package]] name = "wheel" version = "0.41.3" @@ -1598,4 +1679,4 @@ pgsql = ["psycopg2-binary"] [metadata] lock-version = "2.0" python-versions = "^3.8.1" -content-hash = "f7aa2919aca77aa4d1dfcba18c6fc9694a2cc1d5cfd60e7ec991a615251fa86e" +content-hash = "3047372929734b087c9c9419adada7adac1e151f464a1d91961b97da3e3782ec" diff --git a/pyproject.toml b/pyproject.toml index b501ab75..e5666443 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,7 @@ mnemonic = "^0.20" bolt11 = "^2.0.5" black = "23.11.0" pre-commit = "^3.5.0" +websockets = "^12.0" [tool.poetry.extras] pgsql = ["psycopg2-binary"] From 3a64e210eaab2626c873b1b6a8dd1b38aa12bae0 Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Sun, 11 Feb 2024 12:29:00 +0100 Subject: [PATCH 02/45] add test (not working) --- tests/test_mint_operations.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_mint_operations.py b/tests/test_mint_operations.py index 46c05ec0..f363ecf9 100644 --- a/tests/test_mint_operations.py +++ b/tests/test_mint_operations.py @@ -1,3 +1,4 @@ + import pytest import pytest_asyncio @@ -361,3 +362,20 @@ async def test_check_proof_state(wallet1: Wallet, ledger: Ledger): secrets=[p.secret for p in send_proofs] ) assert all([p.state.value == "UNSPENT" for p in proof_states]) + + +# TODO: test keeps running forever, needs to be fixed +# @pytest.mark.asyncio +# async def test_websocket_quote_updates(wallet1: Wallet, ledger: Ledger): +# invoice = await wallet1.request_mint(64) +# ws = websocket.create_connection( +# f"ws://localhost:{SERVER_PORT}/v1/quote/{invoice.id}" +# ) +# await asyncio.sleep(0.1) +# pay_if_regtest(invoice.bolt11) +# await wallet1.mint(64, id=invoice.id) +# await asyncio.sleep(0.1) +# data = str(ws.recv()) +# ws.close() +# n_lines = len(data.split("\n")) +# assert n_lines == 1 From 345aa8ad71e8a58661b066f9cb56dabd57c371c5 Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Thu, 4 Apr 2024 00:38:56 +0200 Subject: [PATCH 03/45] wip: emit events to everyone --- cashu/core/base.py | 40 ++++++++++++++++++++++++++++++---------- cashu/mint/ledger.py | 38 ++++++++++++++++++++++++++------------ cashu/mint/router.py | 24 ++++++++++++++---------- cashu/wallet/cli/cli.py | 3 +-- 4 files changed, 71 insertions(+), 34 deletions(-) diff --git a/cashu/core/base.py b/cashu/core/base.py index f6375f9e..3b04ef3a 100644 --- a/cashu/core/base.py +++ b/cashu/core/base.py @@ -7,8 +7,9 @@ from typing import Any, Dict, List, Optional, Union from loguru import logger -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, root_validator +from ..mint.events.base import LedgerEvent from .crypto.aes import AESCipher from .crypto.b_dhke import hash_to_curve from .crypto.keys import ( @@ -101,12 +102,12 @@ class Proof(BaseModel): time_created: Union[None, str] = "" time_reserved: Union[None, str] = "" derivation_path: Union[None, str] = "" # derivation path of the proof - mint_id: Union[None, str] = ( - None # holds the id of the mint operation that created this proof - ) - melt_id: Union[None, str] = ( - None # holds the id of the melt operation that destroyed this proof - ) + mint_id: Union[ + None, str + ] = None # holds the id of the mint operation that created this proof + melt_id: Union[ + None, str + ] = None # holds the id of the melt operation that destroyed this proof def __init__(self, **data): super().__init__(**data) @@ -225,7 +226,7 @@ class Invoice(BaseModel): time_paid: Union[None, str, int, float] = "" -class MeltQuote(BaseModel): +class MeltQuote(LedgerEvent): quote: str method: str request: str @@ -266,8 +267,12 @@ def from_row(cls, row: Row): proof=row["proof"], ) + def identifier(self) -> str: + """Implementation of the abstract method from LedgerEventManager""" + return self.quote + -class MintQuote(BaseModel): +class MintQuote(LedgerEvent): quote: str method: str request: str @@ -305,6 +310,10 @@ def from_row(cls, row: Row): paid_time=paid_time, ) + def identifier(self) -> str: + """Implementation of the abstract method from LedgerEventManager""" + return self.quote + # ------- API ------- @@ -514,11 +523,22 @@ def __str__(self): return self.name -class ProofState(BaseModel): +class ProofState(LedgerEvent): Y: str state: SpentState witness: Optional[str] = None + @root_validator(pre=True) + def check_witness(cls, values): + state, witness = values.get("state"), values.get("witness") + if witness is not None and state != SpentState.spent: + raise ValueError('Witness can only be set if the spent state is "SPENT"') + return values + + def identifier(self) -> str: + """Implementation of the abstract method from LedgerEventManager""" + return self.Y + class PostCheckStateResponse(BaseModel): states: List[ProofState] = [] diff --git a/cashu/mint/ledger.py b/cashu/mint/ledger.py index 37a83b13..db09b8dc 100644 --- a/cashu/mint/ledger.py +++ b/cashu/mint/ledger.py @@ -6,7 +6,7 @@ import bolt11 from loguru import logger -from cashu.mint.quotes import QuoteQueue +from cashu.mint.events.events import LedgerEventManager from ..core.base import ( DLEQ, @@ -64,7 +64,7 @@ class Ledger(LedgerVerification, LedgerSpendingConditions): asyncio.Lock() ) # holds locks for proofs_pending database keysets: Dict[str, MintKeyset] = {} - quote_queue = QuoteQueue() + events = LedgerEventManager() def __init__( self, @@ -165,7 +165,7 @@ async def _check_pending_proofs_and_melt_quotes(self): logger.info(f"Melt quote {quote.quote} state: failed") # unset pending - await self._unset_proofs_pending(pending_proofs) + await self._unset_proofs_pending(pending_proofs, spent=False) elif payment.pending: logger.info(f"Melt quote {quote.quote} state: pending") pass @@ -328,6 +328,9 @@ async def _invalidate_proofs( await self.crud.invalidate_proof( proof=p, db=self.db, quote_id=quote_id, conn=conn ) + await self.events.submit( + ProofState(Y=p.Y, state=SpentState.spent, witness=p.witness or None) + ) async def _generate_change_promises( self, @@ -457,7 +460,7 @@ async def mint_quote(self, quote_request: PostMintQuoteRequest) -> MintQuote: expiry=expiry, ) await self.crud.store_mint_quote(quote=quote, db=self.db) - await self.quote_queue.submit(quote.quote, quote) + await self.events.submit(quote) return quote @@ -490,7 +493,7 @@ async def get_mint_quote(self, quote_id: str) -> MintQuote: quote.paid = True quote.paid_time = int(time.time()) await self.crud.update_mint_quote(quote=quote, db=self.db) - await self.quote_queue.submit(quote.quote, quote) + await self.events.submit(quote) return quote @@ -540,7 +543,7 @@ async def mint( logger.trace(f"crud: setting quote {quote_id} as issued") quote.issued = True await self.crud.update_mint_quote(quote=quote, db=self.db) - await self.quote_queue.submit(quote.quote, quote) + await self.events.submit(quote) del self.locks[quote_id] return promises @@ -626,7 +629,7 @@ async def melt_quote( expiry=expiry, ) await self.crud.store_melt_quote(quote=quote, db=self.db) - await self.quote_queue.submit(quote.quote, quote) + await self.events.submit(quote) return PostMeltQuoteResponse( quote=quote.quote, @@ -686,7 +689,7 @@ async def get_melt_quote( melt_quote.proof = status.preimage melt_quote.paid_time = int(time.time()) await self.crud.update_melt_quote(quote=melt_quote, db=self.db) - await self.quote_queue.submit(melt_quote.quote, melt_quote) + await self.events.submit(melt_quote) return melt_quote @@ -738,12 +741,12 @@ async def melt_mint_settle_internally(self, melt_quote: MeltQuote) -> MeltQuote: melt_quote.paid = True melt_quote.paid_time = int(time.time()) await self.crud.update_melt_quote(quote=melt_quote, db=self.db) - await self.quote_queue.submit(melt_quote.quote, melt_quote) + await self.events.submit(melt_quote) mint_quote.paid = True mint_quote.paid_time = melt_quote.paid_time await self.crud.update_mint_quote(quote=mint_quote, db=self.db) - await self.quote_queue.submit(mint_quote.quote, mint_quote) + await self.events.submit(mint_quote) return melt_quote @@ -834,7 +837,7 @@ async def melt( melt_quote.paid = True melt_quote.paid_time = int(time.time()) await self.crud.update_melt_quote(quote=melt_quote, db=self.db) - await self.quote_queue.submit(melt_quote.quote, melt_quote) + await self.events.submit(melt_quote) # melt successful, invalidate proofs await self._invalidate_proofs(proofs=proofs, quote_id=melt_quote.quote) @@ -1067,20 +1070,31 @@ async def _set_proofs_pending( await self.crud.set_proof_pending( proof=p, db=self.db, quote_id=quote_id, conn=conn ) + await self.events.submit( + ProofState(Y=p.Y, state=SpentState.pending) + ) except Exception as e: logger.error(f"Failed to set proofs pending: {e}") raise TransactionError("Failed to set proofs pending.") - async def _unset_proofs_pending(self, proofs: List[Proof]) -> None: + async def _unset_proofs_pending(self, proofs: List[Proof], spent=True) -> None: """Deletes proofs from pending table. Args: proofs (List[Proof]): Proofs to delete. + spent (bool): Whether the proofs have been spent or not. Defaults to True. + This should be False if the proofs were NOT invalidated before calling this function. + It is used to emit the unspent state for the proofs (otherwise the spent state is emitted + by the _invalidate_proofs function when the proofs are spent). """ async with self.proofs_pending_lock: async with self.db.connect() as conn: for p in proofs: await self.crud.unset_proof_pending(proof=p, db=self.db, conn=conn) + if not spent: + await self.events.submit( + ProofState(Y=p.Y, state=SpentState.unspent) + ) async def _validate_proofs_pending( self, proofs: List[Proof], conn: Optional[Connection] = None diff --git a/cashu/mint/router.py b/cashu/mint/router.py index 88770311..859ccaa2 100644 --- a/cashu/mint/router.py +++ b/cashu/mint/router.py @@ -1,8 +1,10 @@ from typing import Any, Dict, List -from fastapi import APIRouter, Request, WebSocket, WebSocketDisconnect +from fastapi import APIRouter, Request, WebSocket from loguru import logger +from cashu.mint.events.client import LedgerEventClientManager + from ..core.base import ( GetInfoResponse, KeysetsResponse, @@ -224,16 +226,18 @@ async def get_mint_quote(request: Request, quote: str) -> PostMintQuoteResponse: return resp -@router.websocket("/v1/quote/{quote_id}", name="Quote updates") -@router.websocket("/v1/mint/quote/bolt11/{quote_id}", name="Mint quote updates") -@router.websocket("/v1/melt/quote/bolt11/{quote_id}", name="Melt quote updates") -async def websocket_endpoint(websocket: WebSocket, quote_id: str): - await websocket.accept() +@router.websocket("/v1/ws/checkstate", name="Subscribe to updates") +async def websocket_endpoint(websocket: WebSocket): + client = LedgerEventClientManager(websocket=websocket) + success = ledger.events.add_client(client) + if not success: + await websocket.close() + return try: - async for quote in ledger.quote_queue.watch(quote_id): - await websocket.send_json(quote.dict()) - except WebSocketDisconnect: - pass + await client.start() + except Exception as e: + logger.warning(e) + ledger.events.remove_client(client) @router.post( diff --git a/cashu/wallet/cli/cli.py b/cashu/wallet/cli/cli.py index 4f602baf..94af9252 100644 --- a/cashu/wallet/cli/cli.py +++ b/cashu/wallet/cli/cli.py @@ -422,7 +422,6 @@ async def balance(ctx: Context, verbose): ) @click.option( "--legacy", - "-l", default=False, is_flag=True, help="Print legacy token without mint information.", @@ -738,7 +737,7 @@ async def invoices(ctx, paid: bool, unpaid: bool, pending: bool, mint: bool): return if mint: - await wallet.load_mint() + await wallet.load_mint() paid_arg = None if unpaid: From 7f050350768ba3dd88003cc8e23392a3a0467579 Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Thu, 4 Apr 2024 00:39:55 +0200 Subject: [PATCH 04/45] wip: emit events to everyone --- cashu/core/base.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cashu/core/base.py b/cashu/core/base.py index 3b04ef3a..31cc0c13 100644 --- a/cashu/core/base.py +++ b/cashu/core/base.py @@ -267,6 +267,7 @@ def from_row(cls, row: Row): proof=row["proof"], ) + @property def identifier(self) -> str: """Implementation of the abstract method from LedgerEventManager""" return self.quote @@ -310,6 +311,7 @@ def from_row(cls, row: Row): paid_time=paid_time, ) + @property def identifier(self) -> str: """Implementation of the abstract method from LedgerEventManager""" return self.quote @@ -535,6 +537,7 @@ def check_witness(cls, values): raise ValueError('Witness can only be set if the spent state is "SPENT"') return values + @property def identifier(self) -> str: """Implementation of the abstract method from LedgerEventManager""" return self.Y From 8d714b286942d422dfa191c507e4cc0070ab3f3e Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Fri, 5 Apr 2024 01:54:24 +0200 Subject: [PATCH 05/45] wip, lots of things broken but invoice callback works --- cashu/core/base.py | 2 +- cashu/lightning/base.py | 8 +-- cashu/lightning/fake.py | 62 +++++++++++++----- cashu/mint/crud.py | 105 +++++++++++++++++++++--------- cashu/mint/ledger.py | 12 ++-- cashu/mint/protocols.py | 5 ++ cashu/mint/router.py | 7 +- cashu/wallet/cli/cli.py | 28 ++++++-- cashu/wallet/wallet.py | 60 +++++++++++++---- cashu/wallet/wallet_deprecated.py | 15 ++--- tests/test_mint_db.py | 4 +- 11 files changed, 219 insertions(+), 89 deletions(-) diff --git a/cashu/core/base.py b/cashu/core/base.py index 31cc0c13..c1530997 100644 --- a/cashu/core/base.py +++ b/cashu/core/base.py @@ -9,7 +9,7 @@ from loguru import logger from pydantic import BaseModel, Field, root_validator -from ..mint.events.base import LedgerEvent +from ..mint.events.events import LedgerEvent from .crypto.aes import AESCipher from .crypto.b_dhke import hash_to_curve from .crypto.keys import ( diff --git a/cashu/lightning/base.py b/cashu/lightning/base.py index 083554fd..6c5c0d6d 100644 --- a/cashu/lightning/base.py +++ b/cashu/lightning/base.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from typing import Coroutine, Optional, Union +from typing import AsyncGenerator, Coroutine, Optional, Union from pydantic import BaseModel @@ -118,9 +118,9 @@ async def get_payment_quote( # ) -> InvoiceQuoteResponse: # pass - # @abstractmethod - # def paid_invoices_stream(self) -> AsyncGenerator[str, None]: - # pass + @abstractmethod + def paid_invoices_stream(self) -> AsyncGenerator[str, None]: + pass class Unsupported(Exception): diff --git a/cashu/lightning/fake.py b/cashu/lightning/fake.py index 564c000d..a45b1699 100644 --- a/cashu/lightning/fake.py +++ b/cashu/lightning/fake.py @@ -4,7 +4,7 @@ import random from datetime import datetime from os import urandom -from typing import AsyncGenerator, Dict, Optional, Set +from typing import AsyncGenerator, Dict, List, Optional from bolt11 import ( Bolt11, @@ -30,9 +30,11 @@ class FakeWallet(LightningBackend): fake_btc_price = 1e8 / 1337 - queue: asyncio.Queue[Bolt11] = asyncio.Queue(0) + paid_invoices_queue: asyncio.Queue[Bolt11] = asyncio.Queue(0) payment_secrets: Dict[str, str] = dict() - paid_invoices: Set[str] = set() + created_invoices: List[Bolt11] = [] + paid_invoices_outgoing: List[Bolt11] = [] + paid_invoices_incoming: List[Bolt11] = [] secret: str = "FAKEWALLET SECRET" privkey: str = hashlib.pbkdf2_hmac( "sha256", @@ -52,6 +54,11 @@ def __init__(self, unit: Unit = Unit.sat, **kwargs): async def status(self) -> StatusResponse: return StatusResponse(error_message=None, balance=1337) + async def mark_invoice_paid(self, invoice: Bolt11) -> None: + await asyncio.sleep(1) + self.paid_invoices_incoming.append(invoice) + await self.paid_invoices_queue.put(invoice) + async def create_invoice( self, amount: Amount, @@ -105,8 +112,15 @@ async def create_invoice( tags=tags, ) + if bolt11 not in self.created_invoices: + self.created_invoices.append(bolt11) + else: + raise ValueError("Invoice already created") + payment_request = encode(bolt11, self.privkey) + asyncio.create_task(self.mark_invoice_paid(bolt11)) + return InvoiceResponse( ok=True, checking_id=payment_hash, payment_request=payment_request ) @@ -118,8 +132,11 @@ async def pay_invoice(self, quote: MeltQuote, fee_limit: int) -> PaymentResponse await asyncio.sleep(5) if invoice.payment_hash in self.payment_secrets or settings.fakewallet_brr: - await self.queue.put(invoice) - self.paid_invoices.add(invoice.payment_hash) + if invoice not in self.paid_invoices_outgoing: + self.paid_invoices_outgoing.append(invoice) + else: + raise ValueError("Invoice already paid") + return PaymentResponse( ok=True, checking_id=invoice.payment_hash, @@ -132,20 +149,30 @@ async def pay_invoice(self, quote: MeltQuote, fee_limit: int) -> PaymentResponse ) async def get_invoice_status(self, checking_id: str) -> PaymentStatus: - if settings.fakewallet_stochastic_invoice: - paid = random.random() > 0.7 - return PaymentStatus(paid=paid) - paid = checking_id in self.paid_invoices or settings.fakewallet_brr - return PaymentStatus(paid=paid or None) + is_created_invoice = any( + [checking_id == i.payment_hash for i in self.created_invoices] + ) + if not is_created_invoice: + return PaymentStatus(paid=None) + invoice = next( + i for i in self.created_invoices if i.payment_hash == checking_id + ) + paid = False + if is_created_invoice or ( + settings.fakewallet_brr + or (settings.fakewallet_stochastic_invoice and random.random() > 0.7) + ): + paid = True + + # invoice is paid but not in paid_invoices_incoming yet + # so we add it to the paid_invoices_queue + if paid and invoice not in self.paid_invoices_incoming: + await self.paid_invoices_queue.put(invoice) + return PaymentStatus(paid=paid) async def get_payment_status(self, _: str) -> PaymentStatus: return PaymentStatus(paid=settings.fakewallet_payment_state) - async def paid_invoices_stream(self) -> AsyncGenerator[str, None]: - while True: - value: Bolt11 = await self.queue.get() - yield value.payment_hash - # async def get_invoice_quote(self, bolt11: str) -> InvoiceQuoteResponse: # invoice_obj = decode(bolt11) # assert invoice_obj.amount_msat, "invoice has no amount." @@ -173,3 +200,8 @@ async def get_payment_quote(self, bolt11: str) -> PaymentQuoteResponse: fee=fees.to(self.unit, round="up"), amount=amount.to(self.unit, round="up"), ) + + async def paid_invoices_stream(self) -> AsyncGenerator[str, None]: + while True: + value: Bolt11 = await self.paid_invoices_queue.get() + yield value.payment_hash diff --git a/cashu/mint/crud.py b/cashu/mint/crud.py index 30d30b1c..ffabd793 100644 --- a/cashu/mint/crud.py +++ b/cashu/mint/crud.py @@ -34,7 +34,8 @@ async def get_keyset( derivation_path: str = "", seed: str = "", conn: Optional[Connection] = None, - ) -> List[MintKeyset]: ... + ) -> List[MintKeyset]: + ... @abstractmethod async def get_spent_proofs( @@ -42,7 +43,8 @@ async def get_spent_proofs( *, db: Database, conn: Optional[Connection] = None, - ) -> List[Proof]: ... + ) -> List[Proof]: + ... async def get_proof_used( self, @@ -50,7 +52,8 @@ async def get_proof_used( Y: str, db: Database, conn: Optional[Connection] = None, - ) -> Optional[Proof]: ... + ) -> Optional[Proof]: + ... @abstractmethod async def invalidate_proof( @@ -60,7 +63,8 @@ async def invalidate_proof( proof: Proof, quote_id: Optional[str] = None, conn: Optional[Connection] = None, - ) -> None: ... + ) -> None: + ... @abstractmethod async def get_all_melt_quotes_from_pending_proofs( @@ -68,7 +72,8 @@ async def get_all_melt_quotes_from_pending_proofs( *, db: Database, conn: Optional[Connection] = None, - ) -> List[MeltQuote]: ... + ) -> List[MeltQuote]: + ... @abstractmethod async def get_pending_proofs_for_quote( @@ -77,7 +82,8 @@ async def get_pending_proofs_for_quote( quote_id: str, db: Database, conn: Optional[Connection] = None, - ) -> List[Proof]: ... + ) -> List[Proof]: + ... @abstractmethod async def get_proofs_pending( @@ -86,7 +92,8 @@ async def get_proofs_pending( Ys: List[str], db: Database, conn: Optional[Connection] = None, - ) -> List[Proof]: ... + ) -> List[Proof]: + ... @abstractmethod async def set_proof_pending( @@ -96,7 +103,8 @@ async def set_proof_pending( proof: Proof, quote_id: Optional[str] = None, conn: Optional[Connection] = None, - ) -> None: ... + ) -> None: + ... @abstractmethod async def unset_proof_pending( @@ -105,7 +113,8 @@ async def unset_proof_pending( proof: Proof, db: Database, conn: Optional[Connection] = None, - ) -> None: ... + ) -> None: + ... @abstractmethod async def store_keyset( @@ -114,14 +123,16 @@ async def store_keyset( db: Database, keyset: MintKeyset, conn: Optional[Connection] = None, - ) -> None: ... + ) -> None: + ... @abstractmethod async def get_balance( self, db: Database, conn: Optional[Connection] = None, - ) -> int: ... + ) -> int: + ... @abstractmethod async def store_promise( @@ -135,7 +146,8 @@ async def store_promise( e: str = "", s: str = "", conn: Optional[Connection] = None, - ) -> None: ... + ) -> None: + ... @abstractmethod async def get_promise( @@ -144,7 +156,8 @@ async def get_promise( db: Database, b_: str, conn: Optional[Connection] = None, - ) -> Optional[BlindedSignature]: ... + ) -> Optional[BlindedSignature]: + ... @abstractmethod async def store_mint_quote( @@ -153,16 +166,20 @@ async def store_mint_quote( quote: MintQuote, db: Database, conn: Optional[Connection] = None, - ) -> None: ... + ) -> None: + ... @abstractmethod async def get_mint_quote( self, *, - quote_id: str, + quote_id: Optional[str] = None, + checking_id: Optional[str] = None, + request: Optional[str] = None, db: Database, conn: Optional[Connection] = None, - ) -> Optional[MintQuote]: ... + ) -> Optional[MintQuote]: + ... @abstractmethod async def get_mint_quote_by_request( @@ -171,7 +188,8 @@ async def get_mint_quote_by_request( request: str, db: Database, conn: Optional[Connection] = None, - ) -> Optional[MintQuote]: ... + ) -> Optional[MintQuote]: + ... @abstractmethod async def update_mint_quote( @@ -180,7 +198,8 @@ async def update_mint_quote( quote: MintQuote, db: Database, conn: Optional[Connection] = None, - ) -> None: ... + ) -> None: + ... # @abstractmethod # async def update_mint_quote_paid( @@ -199,17 +218,20 @@ async def store_melt_quote( quote: MeltQuote, db: Database, conn: Optional[Connection] = None, - ) -> None: ... + ) -> None: + ... @abstractmethod async def get_melt_quote( self, *, - quote_id: str, - db: Database, + quote_id: Optional[str] = None, checking_id: Optional[str] = None, + request: Optional[str] = None, + db: Database, conn: Optional[Connection] = None, - ) -> Optional[MeltQuote]: ... + ) -> Optional[MeltQuote]: + ... @abstractmethod async def update_melt_quote( @@ -218,7 +240,8 @@ async def update_melt_quote( quote: MeltQuote, db: Database, conn: Optional[Connection] = None, - ) -> None: ... + ) -> None: + ... class LedgerCrudSqlite(LedgerCrud): @@ -430,17 +453,36 @@ async def store_mint_quote( async def get_mint_quote( self, *, - quote_id: str, + quote_id: Optional[str] = None, + checking_id: Optional[str] = None, + request: Optional[str] = None, db: Database, conn: Optional[Connection] = None, ) -> Optional[MintQuote]: + clauses = [] + values: List[Any] = [] + if quote_id: + clauses.append("quote = ?") + values.append(quote_id) + if checking_id: + clauses.append("checking_id = ?") + values.append(checking_id) + if request: + clauses.append("request = ?") + values.append(request) + if not any(clauses): + raise ValueError("No search criteria") + + where = f"WHERE {' AND '.join(clauses)}" row = await (conn or db).fetchone( f""" SELECT * from {table_with_schema(db, 'mint_quotes')} - WHERE quote = ? + {where} """, - (quote_id,), + tuple(values), ) + if row is None: + return None return MintQuote.from_row(row) if row else None async def get_mint_quote_by_request( @@ -526,10 +568,10 @@ async def store_melt_quote( async def get_melt_quote( self, *, - quote_id: str, - db: Database, + quote_id: Optional[str] = None, checking_id: Optional[str] = None, request: Optional[str] = None, + db: Database, conn: Optional[Connection] = None, ) -> Optional[MeltQuote]: clauses = [] @@ -543,9 +585,10 @@ async def get_melt_quote( if request: clauses.append("request = ?") values.append(request) - where = "" - if clauses: - where = f"WHERE {' AND '.join(clauses)}" + if not any(clauses): + raise ValueError("No search criteria") + where = f"WHERE {' AND '.join(clauses)}" + row = await (conn or db).fetchone( f""" SELECT * from {table_with_schema(db, 'melt_quotes')} diff --git a/cashu/mint/ledger.py b/cashu/mint/ledger.py index db09b8dc..ca9d6da1 100644 --- a/cashu/mint/ledger.py +++ b/cashu/mint/ledger.py @@ -7,6 +7,7 @@ from loguru import logger from cashu.mint.events.events import LedgerEventManager +from cashu.mint.tasks import LedgerTasks from ..core.base import ( DLEQ, @@ -57,7 +58,7 @@ from .verification import LedgerVerification -class Ledger(LedgerVerification, LedgerSpendingConditions): +class Ledger(LedgerVerification, LedgerSpendingConditions, LedgerTasks): backends: Mapping[Method, Mapping[Unit, LightningBackend]] = {} locks: Dict[str, asyncio.Lock] = {} # holds multiprocessing locks proofs_pending_lock: asyncio.Lock = ( @@ -101,6 +102,7 @@ def __init__( async def startup_ledger(self): await self._startup_ledger() await self._check_pending_proofs_and_melt_quotes() + await self.dispatch_listeners() async def _startup_ledger(self): if settings.mint_cache_secrets: @@ -575,9 +577,7 @@ async def melt_quote( # check if there is a mint quote with the same payment request # so that we would be able to handle the transaction internally # and therefore respond with internal transaction fees (0 for now) - mint_quote = await self.crud.get_mint_quote_by_request( - request=request, db=self.db - ) + mint_quote = await self.crud.get_mint_quote(request=request, db=self.db) if mint_quote: assert request == mint_quote.request, "bolt11 requests do not match" assert mint_quote.unit == melt_quote.unit, "units do not match" @@ -668,7 +668,7 @@ async def get_melt_quote( # we only check the state with the backend if there is no associated internal # mint quote for this melt quote - mint_quote = await self.crud.get_mint_quote_by_request( + mint_quote = await self.crud.get_mint_quote( request=melt_quote.request, db=self.db ) @@ -708,7 +708,7 @@ async def melt_mint_settle_internally(self, melt_quote: MeltQuote) -> MeltQuote: """ # first we check if there is a mint quote with the same payment request # so that we can handle the transaction internally without the backend - mint_quote = await self.crud.get_mint_quote_by_request( + mint_quote = await self.crud.get_mint_quote( request=melt_quote.request, db=self.db ) if not mint_quote: diff --git a/cashu/mint/protocols.py b/cashu/mint/protocols.py index 04d24c0c..ff576d96 100644 --- a/cashu/mint/protocols.py +++ b/cashu/mint/protocols.py @@ -4,6 +4,7 @@ from ..core.db import Database from ..lightning.base import LightningBackend from ..mint.crud import LedgerCrud +from .events.events import LedgerEventManager class SupportsKeysets(Protocol): @@ -18,3 +19,7 @@ class SupportsBackends(Protocol): class SupportsDb(Protocol): db: Database crud: LedgerCrud + + +class SupportsEvents(Protocol): + events: LedgerEventManager diff --git a/cashu/mint/router.py b/cashu/mint/router.py index 859ccaa2..a719bf12 100644 --- a/cashu/mint/router.py +++ b/cashu/mint/router.py @@ -3,7 +3,7 @@ from fastapi import APIRouter, Request, WebSocket from loguru import logger -from cashu.mint.events.client import LedgerEventClientManager +from cashu.mint.events.client_manager import LedgerEventClientManager from ..core.base import ( GetInfoResponse, @@ -226,7 +226,7 @@ async def get_mint_quote(request: Request, quote: str) -> PostMintQuoteResponse: return resp -@router.websocket("/v1/ws/checkstate", name="Subscribe to updates") +@router.websocket("/v1/ws", name="Websocket endpoint for subscriptions") async def websocket_endpoint(websocket: WebSocket): client = LedgerEventClientManager(websocket=websocket) success = ledger.events.add_client(client) @@ -235,8 +235,7 @@ async def websocket_endpoint(websocket: WebSocket): return try: await client.start() - except Exception as e: - logger.warning(e) + except Exception: ledger.events.remove_client(client) diff --git a/cashu/wallet/cli/cli.py b/cashu/wallet/cli/cli.py index 94af9252..d29f5bb4 100644 --- a/cashu/wallet/cli/cli.py +++ b/cashu/wallet/cli/cli.py @@ -260,9 +260,23 @@ async def invoice(ctx: Context, amount: float, id: str, split: int, no_check: bo f"Requesting split with {n_splits} * {wallet.unit.str(split)} tokens." ) + paid = False + + def mint_invoice_callback(msg): + nonlocal ctx, wallet, amount, optional_split, paid + print("Received callback for invoice.") + asyncio.run(wallet.mint(int(amount), split=optional_split, id=invoice.id)) + print(" Invoice callback paid.") + print("") + asyncio.run(print_balance(ctx)) + os._exit(0) + # user requests an invoice if amount and not id: - invoice = await wallet.request_mint(amount) + mint_supports_websockets = True + invoice, subscription = await wallet.request_mint_subscription( + amount, callback=mint_invoice_callback + ) if invoice.bolt11: print("") print(f"Pay invoice to mint {wallet.unit.str(amount)}:") @@ -275,20 +289,26 @@ async def invoice(ctx: Context, amount: float, id: str, split: int, no_check: bo ) if no_check: return - check_until = time.time() + 5 * 60 # check for five minutes print("") print( "Checking invoice ...", end="", flush=True, ) + if mint_supports_websockets: + while not paid: + await asyncio.sleep(0.1) + + if not mint_supports_websockets: + check_until = time.time() + 5 * 60 # check for five minutes paid = False while time.time() < check_until and not paid: - time.sleep(3) + await asyncio.sleep(10) try: - await wallet.mint(amount, split=optional_split, id=invoice.id) + # await wallet.mint(amount, split=optional_split, id=invoice.id) paid = True print(" Invoice paid.") + subscription.close() except Exception as e: # TODO: user error codes! if "not paid" in str(e): diff --git a/cashu/wallet/wallet.py b/cashu/wallet/wallet.py index acd41cfd..dc12d064 100644 --- a/cashu/wallet/wallet.py +++ b/cashu/wallet/wallet.py @@ -1,11 +1,12 @@ import base64 import copy import json +import threading import time import uuid from itertools import groupby from posixpath import join -from typing import Dict, List, Optional, Tuple, Union +from typing import Callable, Dict, List, Optional, Tuple, Union import bolt11 import httpx @@ -73,6 +74,7 @@ from .htlc import WalletHTLC from .p2pk import WalletP2PK from .secrets import WalletSecrets +from .subscriptions import SubscriptionManager from .wallet_deprecated import LedgerAPIDeprecated @@ -444,7 +446,7 @@ async def _get_info(self) -> GetInfoResponse: @async_set_httpx_client @async_ensure_mint_loaded - async def mint_quote(self, amount) -> Invoice: + async def mint_quote(self, amount) -> PostMintQuoteResponse: """Requests a mint quote from the server and returns a payment request. Args: @@ -469,16 +471,7 @@ async def mint_quote(self, amount) -> Invoice: # END backwards compatibility < 0.15.0 self.raise_on_error_request(resp) return_dict = resp.json() - mint_response = PostMintQuoteResponse.parse_obj(return_dict) - decoded_invoice = bolt11.decode(mint_response.request) - return Invoice( - amount=amount, - bolt11=mint_response.request, - payment_hash=decoded_invoice.payment_hash, - id=mint_response.quote, - out=False, - time_created=int(time.time()), - ) + return PostMintQuoteResponse.parse_obj(return_dict) @async_set_httpx_client @async_ensure_mint_loaded @@ -816,16 +809,57 @@ async def load_keysets(self) -> None: for keyset in keysets: self.keysets[keyset.id] = keyset + async def request_mint_subscription( + self, amount: int, callback: Callable + ) -> Tuple[Invoice, SubscriptionManager]: + """Request a Lightning invoice for minting tokens. + + Args: + amount (int): Amount for Lightning invoice in satoshis + callback (Optional[Callable], optional): Callback function to be called when the invoice is paid. Defaults to None. + + Returns: + Invoice: Lightning invoice + """ + mint_qoute = await super().mint_quote(amount) + subscriptions = SubscriptionManager(self.url) + threading.Thread(target=subscriptions.connect).start() + subscriptions.subscribe(filters=[mint_qoute.quote], callback=callback) + # return the invoice + decoded_invoice = bolt11.decode(mint_qoute.request) + invoice = Invoice( + amount=amount, + bolt11=mint_qoute.request, + payment_hash=decoded_invoice.payment_hash, + id=mint_qoute.quote, + out=False, + time_created=int(time.time()), + ) + await store_lightning_invoice(db=self.db, invoice=invoice) + return invoice, subscriptions + async def request_mint(self, amount: int) -> Invoice: """Request a Lightning invoice for minting tokens. Args: amount (int): Amount for Lightning invoice in satoshis + callback (Optional[Callable], optional): Callback function to be called when the invoice is paid. Defaults to None. Returns: Invoice: Lightning invoice """ - invoice = await super().mint_quote(amount) + mint_qoute = await super().mint_quote(amount) + + # return the invoice + decoded_invoice = bolt11.decode(mint_qoute.request) + invoice = Invoice( + amount=amount, + bolt11=mint_qoute.request, + payment_hash=decoded_invoice.payment_hash, + id=mint_qoute.quote, + out=False, + time_created=int(time.time()), + ) await store_lightning_invoice(db=self.db, invoice=invoice) return invoice diff --git a/cashu/wallet/wallet_deprecated.py b/cashu/wallet/wallet_deprecated.py index db5e927a..cab7014c 100644 --- a/cashu/wallet/wallet_deprecated.py +++ b/cashu/wallet/wallet_deprecated.py @@ -16,10 +16,10 @@ GetInfoResponse, GetInfoResponse_deprecated, GetMintResponse_deprecated, - Invoice, KeysetsResponse_deprecated, PostMeltRequest_deprecated, PostMeltResponse_deprecated, + PostMintQuoteResponse, PostMintRequest_deprecated, PostMintResponse_deprecated, PostRestoreResponse, @@ -228,7 +228,7 @@ async def _get_keyset_ids_deprecated(self, url: str) -> List[str]: @async_set_httpx_client @async_ensure_mint_loaded_deprecated - async def request_mint_deprecated(self, amount) -> Invoice: + async def request_mint_deprecated(self, amount) -> PostMintQuoteResponse: """Requests a mint from the server and returns Lightning invoice. Args: @@ -246,12 +246,11 @@ async def request_mint_deprecated(self, amount) -> Invoice: return_dict = resp.json() mint_response = GetMintResponse_deprecated.parse_obj(return_dict) decoded_invoice = bolt11.decode(mint_response.pr) - return Invoice( - amount=amount, - bolt11=mint_response.pr, - id=mint_response.hash, - payment_hash=decoded_invoice.payment_hash, - out=False, + return PostMintQuoteResponse( + quote=mint_response.hash, + request=mint_response.pr, + paid=False, + expiry=decoded_invoice.date + (decoded_invoice.expiry or 0), ) @async_set_httpx_client diff --git a/tests/test_mint_db.py b/tests/test_mint_db.py index 92caf1e7..4d802233 100644 --- a/tests/test_mint_db.py +++ b/tests/test_mint_db.py @@ -50,9 +50,7 @@ async def test_mint_quote(wallet1: Wallet, ledger: Ledger): async def test_get_mint_quote_by_request(wallet1: Wallet, ledger: Ledger): invoice = await wallet1.request_mint(128) assert invoice is not None - quote = await ledger.crud.get_mint_quote_by_request( - request=invoice.bolt11, db=ledger.db - ) + quote = await ledger.crud.get_mint_quote(request=invoice.bolt11, db=ledger.db) assert quote is not None assert quote.quote == invoice.id assert quote.amount == 128 From 5aea9a415339ce99df54e79c54e39c962be60907 Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Fri, 5 Apr 2024 09:25:06 +0200 Subject: [PATCH 06/45] wip --- cashu/mint/ledger.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cashu/mint/ledger.py b/cashu/mint/ledger.py index ca9d6da1..23e29dca 100644 --- a/cashu/mint/ledger.py +++ b/cashu/mint/ledger.py @@ -539,12 +539,14 @@ async def mint( if quote.expiry: assert quote.expiry > int(time.time()), "quote expired" - promises = await self._generate_promises(outputs) - logger.trace("generated promises") - logger.trace(f"crud: setting quote {quote_id} as issued") quote.issued = True await self.crud.update_mint_quote(quote=quote, db=self.db) + + promises = await self._generate_promises(outputs) + logger.trace("generated promises") + + # submit the quote update to the event manager await self.events.submit(quote) del self.locks[quote_id] From 0022287e44db77465a2a1baf917ea6280e731ae8 Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Fri, 5 Apr 2024 14:00:08 +0200 Subject: [PATCH 07/45] add wip files --- cashu/core/json_rpc/base.py | 74 +++++++++++++++ cashu/mint/events/client_manager.py | 140 ++++++++++++++++++++++++++++ cashu/mint/events/events.py | 49 ++++++++++ cashu/wallet/subscriptions.py | 86 +++++++++++++++++ 4 files changed, 349 insertions(+) create mode 100644 cashu/core/json_rpc/base.py create mode 100644 cashu/mint/events/client_manager.py create mode 100644 cashu/mint/events/events.py create mode 100644 cashu/wallet/subscriptions.py diff --git a/cashu/core/json_rpc/base.py b/cashu/core/json_rpc/base.py new file mode 100644 index 00000000..9682f031 --- /dev/null +++ b/cashu/core/json_rpc/base.py @@ -0,0 +1,74 @@ +from enum import Enum +from typing import List + +from pydantic import BaseModel, Field + +from ..settings import settings + + +class JSONRPCRequest(BaseModel): + jsonrpc: str = "2.0" + id: int + method: str + params: dict + + +class JSONRPCResponse(BaseModel): + jsonrpc: str = "2.0" + result: dict + id: int + + +class JSONRPCNotification(BaseModel): + jsonrpc: str = "2.0" + method: str + params: dict + + +class JSONRPCErrorCode(Enum): + PARSE_ERROR = -32700 + INVALID_REQUEST = -32600 + METHOD_NOT_FOUND = -32601 + INVALID_PARAMS = -32602 + INTERNAL_ERROR = -32603 + SERVER_ERROR = -32000 + APPLICATION_ERROR = -32099 + SYSTEM_ERROR = -32098 + TRANSPORT_ERROR = -32097 + + +class JSONRPCError(BaseModel): + code: JSONRPCErrorCode + message: str + + +class JSONRPCErrorResponse(BaseModel): + jsonrpc: str = "2.0" + error: JSONRPCError + id: int + + +# Cashu Websocket protocol + + +class JSONRPCMethods(Enum): + SUBSCRIBE = "subscribe" + UNSUBSCRIBE = "unsubscribe" + + +class JSONRPCStatus(Enum): + OK = "OK" + + +class JSONRPCSubscribeParams(BaseModel): + filters: List[str] = Field(..., max_length=settings.mint_max_request_length) + subId: str + + +class JSONRPCUnubscribeParams(BaseModel): + subId: str + + +class JSONRRPCSubscribeResponse(BaseModel): + status: JSONRPCStatus + subId: str diff --git a/cashu/mint/events/client_manager.py b/cashu/mint/events/client_manager.py new file mode 100644 index 00000000..c35c8dbf --- /dev/null +++ b/cashu/mint/events/client_manager.py @@ -0,0 +1,140 @@ +import json +from typing import List, Union + +from fastapi import WebSocket +from loguru import logger + +from ...core.json_rpc.base import ( + JSONRPCError, + JSONRPCErrorCode, + JSONRPCErrorResponse, + JSONRPCMethods, + JSONRPCNotification, + JSONRPCRequest, + JSONRPCResponse, + JSONRPCStatus, + JSONRPCSubscribeParams, + JSONRPCUnubscribeParams, + JSONRRPCSubscribeResponse, +) + + +class LedgerEventClientManager: + websocket: WebSocket + subscriptions: dict[str, List[str]] = {} # filter -> List[subId] + max_subscriptions = 100 + + def __init__(self, websocket: WebSocket): + self.websocket = websocket + + async def start(self): + await self.websocket.accept() + while True: + json_data = await self.websocket.receive_text() + + try: + data = json.loads(json_data) + except json.JSONDecodeError as e: + logger.error(f"Error decoding JSON: {e}") + resp = JSONRPCErrorResponse( + error=JSONRPCError( + code=JSONRPCErrorCode.PARSE_ERROR, + message=f"Error: {e}", + ), + id=0, + ) + await self._send_msg(resp) + continue + + try: + req = JSONRPCRequest.parse_obj(data) + except Exception as e: + resp = JSONRPCErrorResponse( + error=JSONRPCError( + code=JSONRPCErrorCode.INVALID_REQUEST, + message=f"Error: {e}", + ), + id=0, + ) + await self._send_msg(resp) + logger.warning(f"Error handling websocket message: {e}") + continue + + # check if method is in the enum + try: + JSONRPCMethods(req.method) + except ValueError: + resp = JSONRPCErrorResponse( + error=JSONRPCError( + code=JSONRPCErrorCode.METHOD_NOT_FOUND, + message=f"Method not found: {req.method}", + ), + id=req.id, + ) + await self._send_msg(resp) + continue + try: + resp = await self._handle_request(req) + except Exception as e: + resp = JSONRPCErrorResponse( + error=JSONRPCError( + code=JSONRPCErrorCode.INTERNAL_ERROR, + message=f"Error: {e}", + ), + id=req.id, + ) + await self._send_msg(resp) + + async def _handle_request(self, data: JSONRPCRequest) -> JSONRPCResponse: + logger.info(f"Received message: {data}") + if data.method == JSONRPCMethods.SUBSCRIBE.value: + params = JSONRPCSubscribeParams.parse_obj(data.params) + self.add_subscription(params.filters, params.subId) + result = JSONRRPCSubscribeResponse( + status=JSONRPCStatus.OK, + subId=params.subId, + ) + return JSONRPCResponse(result=result.dict(), id=data.id) + elif data.method == JSONRPCMethods.UNSUBSCRIBE.value: + params = JSONRPCUnubscribeParams.parse_obj(data.params) + self.remove_subscription(params.subId) + result = JSONRRPCSubscribeResponse( + status=JSONRPCStatus.OK, + subId=params.subId, + ) + return JSONRPCResponse(result=result.dict(), id=data.id) + else: + raise ValueError(f"Invalid method: {data.method}") + + async def _send_obj(self, data: dict, subId: str): + logger.info(f"Sending object: {data}") + method = JSONRPCMethods.SUBSCRIBE.value + data.update({"subId": subId}) + params = data + resp = JSONRPCNotification( + method=method, + params=params, + ) + await self._send_msg(resp) + + async def _send_msg( + self, data: Union[JSONRPCResponse, JSONRPCNotification, JSONRPCErrorResponse] + ): + logger.info(f"Sending message: {data}") + await self.websocket.send_text(data.json()) + + def add_subscription(self, filters: List[str], subId: str) -> None: + if len(self.subscriptions) >= self.max_subscriptions: + raise ValueError("Max subscriptions reached") + for filter in filters: + if filter not in self.subscriptions: + self.subscriptions[filter] = [] + self.subscriptions[filter].append(subId) + + def remove_subscription(self, subId: str) -> None: + for filter, subs in self.subscriptions.items(): + for sub in subs: + if sub == subId: + self.subscriptions[filter].remove(sub) + return + raise ValueError(f"Subscription not found: {subId}") diff --git a/cashu/mint/events/events.py b/cashu/mint/events/events.py new file mode 100644 index 00000000..e0656fb9 --- /dev/null +++ b/cashu/mint/events/events.py @@ -0,0 +1,49 @@ +from abc import ABC, abstractmethod + +from pydantic import BaseModel + +from .client_manager import LedgerEventClientManager + + +class LedgerEvent(ABC, BaseModel): + """AbstractBaseClass for BaseModels that can be sent to the + LedgerEventManager for broadcasting subscription events to clients. + """ + + @property + @abstractmethod + def identifier(self) -> str: + pass + + +class LedgerEventManager: + """LedgerEventManager is a subscription service from the mint + for client websockets that subscribe to event updates. + + Yields: + _type_: Union[MintQuote, MeltQuote] + """ + + clients: list[LedgerEventClientManager] = [] + + MAX_CLIENTS = 1000 + + def add_client(self, client: LedgerEventClientManager) -> bool: + if len(self.clients) >= self.MAX_CLIENTS: + return False + self.clients.append(client) + return True + + def remove_client(self, client: LedgerEventClientManager) -> None: + self.clients.remove(client) + + def serialize_event(self, event: LedgerEvent) -> dict: + return event.dict(exclude_unset=True, exclude_none=True) + + async def submit(self, event: LedgerEvent) -> None: + if not isinstance(event, LedgerEvent): + raise ValueError(f"Unsupported event object type {type(event)}") + + for client in self.clients: + for sub in client.subscriptions.get(event.identifier, []): + await client._send_obj(self.serialize_event(event), subId=sub) diff --git a/cashu/wallet/subscriptions.py b/cashu/wallet/subscriptions.py new file mode 100644 index 00000000..faf864cb --- /dev/null +++ b/cashu/wallet/subscriptions.py @@ -0,0 +1,86 @@ +import time +from typing import Callable, List +from urllib.parse import urlparse + +from loguru import logger +from websocket._app import WebSocketApp + +from ..core.crypto.keys import random_hash +from ..core.json_rpc.base import ( + JSONRPCNotification, + JSONRPCRequest, + JSONRPCResponse, + JSONRPCSubscribeParams, +) + + +class SubscriptionManager: + url: str + websocket: WebSocketApp + id_counter: int = 0 + callback_map: dict[str, Callable] = {} + + def __init__(self, url: str): + # parse hostname from url with urlparse + hostname = urlparse(url).hostname + port = urlparse(url).port + if port: + hostname = f"{hostname}:{port}" + scheme = urlparse(url).scheme + ws_scheme = "wss" if scheme == "https" else "ws" + ws_url = f"{ws_scheme}://{hostname}/v1/ws" + self.url = ws_url + self.websocket = WebSocketApp(ws_url, on_message=self._on_message) + + def _on_message(self, ws, message): + try: + msg = JSONRPCResponse.parse_raw(message) + print(msg) + return + except Exception: + pass + + try: + msg = JSONRPCNotification.parse_raw(message) + logger.debug(f"Received notification: {msg}") + self.callback_map[msg.params["subId"]](msg) + return + except Exception: + pass + + logger.error(f"Error parsing message: {message}") + + def connect(self): + self.websocket.run_forever(ping_interval=10, ping_timeout=5) + + def close(self): + self.websocket.close() + + # async def listen(self): + # while True: + # await asyncio.sleep(0) + # try: + # msg = self.websocket.recv() + # print(msg) + # except WebSocketConnectionClosedException: + # print("Connection closed") + # break + # except Exception as e: + # print(f"Error receiving message: {e}") + # break + + def wait_until_connected(self): + while not self.websocket.sock or not self.websocket.sock.connected: + time.sleep(0.025) + + def subscribe(self, filters: List[str], callback: Callable): + self.wait_until_connected() + subId = random_hash() + req = JSONRPCRequest( + method="subscribe", + params=JSONRPCSubscribeParams(filters=filters, subId=subId).dict(), + id=self.id_counter, + ) + self.websocket.send(req.json()) + self.id_counter += 1 + self.callback_map[subId] = callback From 41a85dffd549107f5a7fce1795dd549c21aea832 Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Fri, 5 Apr 2024 18:56:30 +0200 Subject: [PATCH 08/45] tests almost passing --- cashu/core/json_rpc/base.py | 5 ++ cashu/lightning/blink.py | 12 +--- cashu/lightning/fake.py | 44 ++++++++---- cashu/lightning/lnbits.py | 5 +- cashu/lightning/strike.py | 5 +- cashu/mint/events/client_manager.py | 8 +-- cashu/wallet/cli/cli.py | 103 +++++++++++++++++++--------- cashu/wallet/subscriptions.py | 18 ++--- cashu/wallet/wallet.py | 4 +- tests/test_wallet_cli.py | 4 ++ 10 files changed, 128 insertions(+), 80 deletions(-) diff --git a/cashu/core/json_rpc/base.py b/cashu/core/json_rpc/base.py index 9682f031..87514986 100644 --- a/cashu/core/json_rpc/base.py +++ b/cashu/core/json_rpc/base.py @@ -69,6 +69,11 @@ class JSONRPCUnubscribeParams(BaseModel): subId: str +class JSONRPCNotficationParams(BaseModel): + subId: str + payload: dict + + class JSONRRPCSubscribeResponse(BaseModel): status: JSONRPCStatus subId: str diff --git a/cashu/lightning/blink.py b/cashu/lightning/blink.py index 5c9f0dc4..b334ccef 100644 --- a/cashu/lightning/blink.py +++ b/cashu/lightning/blink.py @@ -1,8 +1,7 @@ # type: ignore -import asyncio import json import math -from typing import Dict, Optional, Union +from typing import AsyncGenerator, Dict, Optional, Union import bolt11 import httpx @@ -450,10 +449,5 @@ async def get_payment_quote(self, bolt11: str) -> PaymentQuoteResponse: amount=amount.to(self.unit, round="up"), ) - -async def main(): - pass - - -if __name__ == "__main__": - asyncio.run(main()) + async def paid_invoices_stream(self) -> AsyncGenerator[str, None]: + raise NotImplementedError("paid_invoices_stream not implemented") diff --git a/cashu/lightning/fake.py b/cashu/lightning/fake.py index a45b1699..a6ce7248 100644 --- a/cashu/lightning/fake.py +++ b/cashu/lightning/fake.py @@ -55,10 +55,21 @@ async def status(self) -> StatusResponse: return StatusResponse(error_message=None, balance=1337) async def mark_invoice_paid(self, invoice: Bolt11) -> None: - await asyncio.sleep(1) + await asyncio.sleep(2) self.paid_invoices_incoming.append(invoice) await self.paid_invoices_queue.put(invoice) + def create_dummy_bolt11(self, payment_hash: str) -> Bolt11: + tags = Tags() + tags.add(TagChar.payment_hash, payment_hash) + tags.add(TagChar.payment_secret, urandom(32).hex()) + return Bolt11( + currency="bc", + amount_msat=MilliSatoshi(1337), + date=int(datetime.now().timestamp()), + tags=tags, + ) + async def create_invoice( self, amount: Amount, @@ -119,7 +130,8 @@ async def create_invoice( payment_request = encode(bolt11, self.privkey) - asyncio.create_task(self.mark_invoice_paid(bolt11)) + if settings.fakewallet_brr: + asyncio.create_task(self.mark_invoice_paid(bolt11)) return InvoiceResponse( ok=True, checking_id=payment_hash, payment_request=payment_request @@ -149,25 +161,27 @@ async def pay_invoice(self, quote: MeltQuote, fee_limit: int) -> PaymentResponse ) async def get_invoice_status(self, checking_id: str) -> PaymentStatus: - is_created_invoice = any( - [checking_id == i.payment_hash for i in self.created_invoices] - ) - if not is_created_invoice: - return PaymentStatus(paid=None) - invoice = next( - i for i in self.created_invoices if i.payment_hash == checking_id - ) + # is_created_invoice = any( + # [checking_id == i.payment_hash for i in self.created_invoices] + # ) + # if not is_created_invoice: + # return PaymentStatus(paid=None) + # invoice = next( + # i for i in self.created_invoices if i.payment_hash == checking_id + # ) paid = False - if is_created_invoice or ( - settings.fakewallet_brr - or (settings.fakewallet_stochastic_invoice and random.random() > 0.7) + if settings.fakewallet_brr or ( + settings.fakewallet_stochastic_invoice and random.random() > 0.7 ): paid = True # invoice is paid but not in paid_invoices_incoming yet # so we add it to the paid_invoices_queue - if paid and invoice not in self.paid_invoices_incoming: - await self.paid_invoices_queue.put(invoice) + # if paid and invoice not in self.paid_invoices_incoming: + if paid: + await self.paid_invoices_queue.put( + self.create_dummy_bolt11(payment_hash=checking_id) + ) return PaymentStatus(paid=paid) async def get_payment_status(self, _: str) -> PaymentStatus: diff --git a/cashu/lightning/lnbits.py b/cashu/lightning/lnbits.py index 174236ef..5eb6800d 100644 --- a/cashu/lightning/lnbits.py +++ b/cashu/lightning/lnbits.py @@ -1,5 +1,5 @@ # type: ignore -from typing import Optional +from typing import AsyncGenerator, Optional import httpx from bolt11 import ( @@ -179,3 +179,6 @@ async def get_payment_quote(self, bolt11: str) -> PaymentQuoteResponse: fee=fees.to(self.unit, round="up"), amount=amount.to(self.unit, round="up"), ) + + async def paid_invoices_stream(self) -> AsyncGenerator[str, None]: + raise NotImplementedError("paid_invoices_stream not implemented") diff --git a/cashu/lightning/strike.py b/cashu/lightning/strike.py index 1824c790..2949138b 100644 --- a/cashu/lightning/strike.py +++ b/cashu/lightning/strike.py @@ -1,6 +1,6 @@ # type: ignore import secrets -from typing import Dict, Optional +from typing import AsyncGenerator, Dict, Optional import httpx @@ -195,3 +195,6 @@ async def get_payment_status(self, checking_id: str) -> PaymentStatus: fee_msat=data["details"]["fee"], preimage=data["preimage"], ) + + async def paid_invoices_stream(self) -> AsyncGenerator[str, None]: + raise NotImplementedError("paid_invoices_stream not implemented") diff --git a/cashu/mint/events/client_manager.py b/cashu/mint/events/client_manager.py index c35c8dbf..0dd71ad6 100644 --- a/cashu/mint/events/client_manager.py +++ b/cashu/mint/events/client_manager.py @@ -9,6 +9,7 @@ JSONRPCErrorCode, JSONRPCErrorResponse, JSONRPCMethods, + JSONRPCNotficationParams, JSONRPCNotification, JSONRPCRequest, JSONRPCResponse, @@ -108,12 +109,9 @@ async def _handle_request(self, data: JSONRPCRequest) -> JSONRPCResponse: async def _send_obj(self, data: dict, subId: str): logger.info(f"Sending object: {data}") - method = JSONRPCMethods.SUBSCRIBE.value - data.update({"subId": subId}) - params = data resp = JSONRPCNotification( - method=method, - params=params, + method=JSONRPCMethods.SUBSCRIBE.value, + params=JSONRPCNotficationParams(subId=subId, payload=data).dict(), ) await self._send_msg(resp) diff --git a/cashu/wallet/cli/cli.py b/cashu/wallet/cli/cli.py index d29f5bb4..ef6b4ed4 100644 --- a/cashu/wallet/cli/cli.py +++ b/cashu/wallet/cli/cli.py @@ -9,14 +9,15 @@ from operator import itemgetter from os import listdir from os.path import isdir, join -from typing import Optional +from typing import Optional, Union import click from click import Context from loguru import logger -from ...core.base import Invoice, TokenV3, Unit +from ...core.base import Invoice, MintQuote, TokenV3, Unit from ...core.helpers import sum_proofs +from ...core.json_rpc.base import JSONRPCNotficationParams from ...core.logging import configure_logger from ...core.settings import settings from ...nostr.client.client import NostrClient @@ -44,6 +45,7 @@ send, ) from ..nostr import receive_nostr, send_nostr +from ..subscriptions import SubscriptionManager class NaturalOrderGroup(click.Group): @@ -261,15 +263,41 @@ async def invoice(ctx: Context, amount: float, id: str, split: int, no_check: bo ) paid = False - - def mint_invoice_callback(msg): - nonlocal ctx, wallet, amount, optional_split, paid - print("Received callback for invoice.") - asyncio.run(wallet.mint(int(amount), split=optional_split, id=invoice.id)) - print(" Invoice callback paid.") - print("") - asyncio.run(print_balance(ctx)) - os._exit(0) + invoice_nonlocal: Union[None, Invoice] = None + subscription_nonlocal: Union[None, SubscriptionManager] = None + + def mint_invoice_callback(msg: JSONRPCNotficationParams): + nonlocal \ + ctx, \ + wallet, \ + amount, \ + optional_split, \ + paid, \ + invoice_nonlocal, \ + subscription_nonlocal + logger.trace(f"Received callback: {msg}") + if paid: + return + try: + quote = MintQuote.parse_obj(msg.payload) + except Exception: + return + logger.debug(f"Received callback for quote: {quote}") + if ( + quote.paid + and not quote.issued + and quote.request == invoice.bolt11 + and msg.subId in subscription.callback_map.keys() + ): + try: + asyncio.run( + wallet.mint(int(amount), split=optional_split, id=invoice.id) + ) + except Exception as e: + print(f"Error during mint: {str(e)}") + return + # set paid so we won't react to any more callbacks + paid = True # user requests an invoice if amount and not id: @@ -277,6 +305,7 @@ def mint_invoice_callback(msg): invoice, subscription = await wallet.request_mint_subscription( amount, callback=mint_invoice_callback ) + invoice_nonlocal, subscription_nonlocal = invoice, subscription if invoice.bolt11: print("") print(f"Pay invoice to mint {wallet.unit.str(amount)}:") @@ -299,33 +328,39 @@ def mint_invoice_callback(msg): while not paid: await asyncio.sleep(0.1) - if not mint_supports_websockets: - check_until = time.time() + 5 * 60 # check for five minutes - paid = False - while time.time() < check_until and not paid: - await asyncio.sleep(10) - try: - # await wallet.mint(amount, split=optional_split, id=invoice.id) - paid = True - print(" Invoice paid.") - subscription.close() - except Exception as e: - # TODO: user error codes! - if "not paid" in str(e): - print(".", end="", flush=True) - continue - else: - print(f"Error: {str(e)}") - if not paid: - print("\n") - print( - "Invoice is not paid yet, stopping check. Use the command above to" - " recheck after the invoice has been paid." - ) + # we still check manually every 10 seconds + check_until = time.time() + 5 * 60 # check for five minutes + paid = False + while time.time() < check_until and not paid: + await asyncio.sleep(10) + try: + # await wallet.mint(amount, split=optional_split, id=invoice.id) + paid = True + except Exception as e: + # TODO: user error codes! + if "not paid" in str(e): + print(".", end="", flush=True) + continue + else: + print(f"Error: {str(e)}") + if not paid: + print("\n") + print( + "Invoice is not paid yet, stopping check. Use the command above to" + " recheck after the invoice has been paid." + ) # user paid invoice and want to check it elif amount and id: await wallet.mint(amount, split=optional_split, id=id) + + # close open subscriptions so we can exit + try: + subscription.close() + except Exception: + pass + print(" Invoice paid.") + print("") await print_balance(ctx) return diff --git a/cashu/wallet/subscriptions.py b/cashu/wallet/subscriptions.py index faf864cb..2a132cef 100644 --- a/cashu/wallet/subscriptions.py +++ b/cashu/wallet/subscriptions.py @@ -7,6 +7,7 @@ from ..core.crypto.keys import random_hash from ..core.json_rpc.base import ( + JSONRPCNotficationParams, JSONRPCNotification, JSONRPCRequest, JSONRPCResponse, @@ -42,8 +43,9 @@ def _on_message(self, ws, message): try: msg = JSONRPCNotification.parse_raw(message) + params = JSONRPCNotficationParams.parse_obj(msg.params) logger.debug(f"Received notification: {msg}") - self.callback_map[msg.params["subId"]](msg) + self.callback_map[params.subId](params) return except Exception: pass @@ -54,21 +56,9 @@ def connect(self): self.websocket.run_forever(ping_interval=10, ping_timeout=5) def close(self): + self.websocket.keep_running = False self.websocket.close() - # async def listen(self): - # while True: - # await asyncio.sleep(0) - # try: - # msg = self.websocket.recv() - # print(msg) - # except WebSocketConnectionClosedException: - # print("Connection closed") - # break - # except Exception as e: - # print(f"Error receiving message: {e}") - # break - def wait_until_connected(self): while not self.websocket.sock or not self.websocket.sock.connected: time.sleep(0.025) diff --git a/cashu/wallet/wallet.py b/cashu/wallet/wallet.py index dc12d064..2bede2e7 100644 --- a/cashu/wallet/wallet.py +++ b/cashu/wallet/wallet.py @@ -823,7 +823,9 @@ async def request_mint_subscription( """ mint_qoute = await super().mint_quote(amount) subscriptions = SubscriptionManager(self.url) - threading.Thread(target=subscriptions.connect).start() + threading.Thread( + target=subscriptions.connect, name="SubscriptionManager", daemon=True + ).start() subscriptions.subscribe(filters=[mint_qoute.quote], callback=callback) # return the invoice decoded_invoice = bolt11.decode(mint_qoute.request) diff --git a/tests/test_wallet_cli.py b/tests/test_wallet_cli.py index 884a9059..59fc5176 100644 --- a/tests/test_wallet_cli.py +++ b/tests/test_wallet_cli.py @@ -175,6 +175,7 @@ def test_invoice_with_split(mint, cli_prefix): wallet = asyncio.run(init_wallet()) assert wallet.proof_amounts.count(1) >= 10 + @pytest.mark.skipif(not is_fake, reason="only on fakewallet") def test_invoices_with_minting(cli_prefix): # arrange @@ -223,6 +224,7 @@ def test_invoices_without_minting(cli_prefix): assert get_invoice_from_invoices_command(result.output)["ID"] == invoice.id assert get_invoice_from_invoices_command(result.output)["Paid"] == str(invoice.paid) + @pytest.mark.skipif(not is_fake, reason="only on fakewallet") def test_invoices_with_onlypaid_option(cli_prefix): # arrange @@ -263,6 +265,7 @@ def test_invoices_with_onlypaid_option_without_minting(cli_prefix): assert result.exit_code == 0 assert "No invoices found." in result.output + @pytest.mark.skipif(not is_fake, reason="only on fakewallet") def test_invoices_with_onlyunpaid_option(cli_prefix): # arrange @@ -322,6 +325,7 @@ def test_invoices_with_both_onlypaid_and_onlyunpaid_options(cli_prefix): in result.output ) + @pytest.mark.skipif(not is_fake, reason="only on fakewallet") def test_invoices_with_pending_option(cli_prefix): # arrange From f5a7075fdb712928a3158a10937add06532ba660 Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Wed, 10 Apr 2024 12:39:01 +0200 Subject: [PATCH 09/45] add task --- cashu/mint/tasks.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 cashu/mint/tasks.py diff --git a/cashu/mint/tasks.py b/cashu/mint/tasks.py new file mode 100644 index 00000000..56cfc1d6 --- /dev/null +++ b/cashu/mint/tasks.py @@ -0,0 +1,38 @@ +import asyncio +from typing import Mapping + +from loguru import logger + +from ..core.base import Method, Unit +from ..core.db import Database +from ..lightning.base import LightningBackend +from ..mint.crud import LedgerCrud +from .events.events import LedgerEventManager +from .protocols import SupportsBackends, SupportsDb, SupportsEvents + + +class LedgerTasks(SupportsDb, SupportsBackends, SupportsEvents): + backends: Mapping[Method, Mapping[Unit, LightningBackend]] = {} + db: Database + crud: LedgerCrud + events: LedgerEventManager + + async def dispatch_listeners(self) -> None: + for method, unitbackends in self.backends.items(): + for unit, backend in unitbackends.items(): + logger.debug( + f"Dispatching backend invoice listener for {method} {unit} {backend.__class__.__name__}" + ) + asyncio.create_task(self.invoice_listener(backend)) + + async def invoice_listener(self, backend: LightningBackend) -> None: + async for checking_id in backend.paid_invoices_stream(): + await self.invoice_callback_dispatcher(checking_id) + + async def invoice_callback_dispatcher(self, checking_id: str) -> None: + logger.success(f"invoice callback dispatcher: {checking_id}") + quote = await self.crud.get_mint_quote(checking_id=checking_id, db=self.db) + if not quote: + logger.error(f"Quote not found for {checking_id}") + return + await self.events.submit(quote) From e07ce881e4da3bc696c811f3df0d4b0fe3babaad Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Sun, 14 Apr 2024 11:14:12 +0200 Subject: [PATCH 10/45] refactor nut constants --- cashu/core/base.py | 3 ++ cashu/core/nuts.py | 12 ++++++++ cashu/mint/features.py | 63 +++++++++++++++++++++++++++++++++++++++++ cashu/mint/ledger.py | 8 +++--- cashu/mint/router.py | 41 +-------------------------- cashu/wallet/cli/cli.py | 8 +++--- 6 files changed, 87 insertions(+), 48 deletions(-) create mode 100644 cashu/core/nuts.py create mode 100644 cashu/mint/features.py diff --git a/cashu/core/base.py b/cashu/core/base.py index 03e4a129..5c4e63a9 100644 --- a/cashu/core/base.py +++ b/cashu/core/base.py @@ -327,6 +327,9 @@ class GetInfoResponse(BaseModel): motd: Optional[str] = None nuts: Optional[Dict[int, Dict[str, Any]]] = None + def supports(self, nut: int) -> Optional[bool]: + return nut in self.nuts if self.nuts else None + class GetInfoResponse_deprecated(BaseModel): name: Optional[str] = None diff --git a/cashu/core/nuts.py b/cashu/core/nuts.py new file mode 100644 index 00000000..90996e76 --- /dev/null +++ b/cashu/core/nuts.py @@ -0,0 +1,12 @@ +SWAP_NUT = 3 +MINT_NUT = 4 +MELT_NUT = 5 +INFO_NUT = 6 +STATE_NUT = 7 +FEE_RETURN_NUT = 8 +RESTORE_NUT = 9 +SCRIPT_NUT = 10 +P2PK_NUT = 11 +DLEQ_NUT = 12 +DETERMINSTIC_SECRETS_NUT = 13 +WEBSOCKETS_NUT = 14 diff --git a/cashu/mint/features.py b/cashu/mint/features.py new file mode 100644 index 00000000..07fcf96d --- /dev/null +++ b/cashu/mint/features.py @@ -0,0 +1,63 @@ +from typing import Any, Dict, List + +from fastapi import APIRouter + +from ..core.base import ( + MintMeltMethodSetting, +) +from ..core.nuts import ( + DLEQ_NUT, + FEE_RETURN_NUT, + MELT_NUT, + MINT_NUT, + P2PK_NUT, + RESTORE_NUT, + SCRIPT_NUT, + STATE_NUT, + WEBSOCKETS_NUT, +) +from ..core.settings import settings +from ..mint.startup import ledger + +router: APIRouter = APIRouter() + + +class LedgerFeatures: + def mint_features(self) -> Dict[int, Dict[str, Any]]: + # determine all method-unit pairs + method_settings: Dict[int, List[MintMeltMethodSetting]] = {} + for nut in [MINT_NUT, MELT_NUT]: + method_settings[nut] = [] + for method, unit_dict in ledger.backends.items(): + for unit in unit_dict.keys(): + setting = MintMeltMethodSetting(method=method.name, unit=unit.name) + + if nut == MINT_NUT and settings.mint_max_peg_in: + setting.max_amount = settings.mint_max_peg_in + setting.min_amount = 0 + elif nut == MELT_NUT and settings.mint_max_peg_out: + setting.max_amount = settings.mint_max_peg_out + setting.min_amount = 0 + + method_settings[nut].append(setting) + + supported_dict = dict(supported=True) + + mint_features: Dict[int, Dict[str, Any]] = { + MINT_NUT: dict( + methods=method_settings[MINT_NUT], + disabled=settings.mint_peg_out_only, + ), + MELT_NUT: dict( + methods=method_settings[MELT_NUT], + disabled=False, + ), + STATE_NUT: supported_dict, + FEE_RETURN_NUT: supported_dict, + RESTORE_NUT: supported_dict, + SCRIPT_NUT: supported_dict, + P2PK_NUT: supported_dict, + DLEQ_NUT: supported_dict, + WEBSOCKETS_NUT: supported_dict, + } + return mint_features diff --git a/cashu/mint/ledger.py b/cashu/mint/ledger.py index 19623409..afc88d05 100644 --- a/cashu/mint/ledger.py +++ b/cashu/mint/ledger.py @@ -6,9 +6,6 @@ import bolt11 from loguru import logger -from cashu.mint.events.events import LedgerEventManager -from cashu.mint.tasks import LedgerTasks - from ..core.base import ( DLEQ, Amount, @@ -56,10 +53,13 @@ ) from ..mint.crud import LedgerCrudSqlite from .conditions import LedgerSpendingConditions +from .events.events import LedgerEventManager +from .features import LedgerFeatures +from .tasks import LedgerTasks from .verification import LedgerVerification -class Ledger(LedgerVerification, LedgerSpendingConditions, LedgerTasks): +class Ledger(LedgerVerification, LedgerSpendingConditions, LedgerTasks, LedgerFeatures): backends: Mapping[Method, Mapping[Unit, LightningBackend]] = {} locks: Dict[str, asyncio.Lock] = {} # holds multiprocessing locks proofs_pending_lock: asyncio.Lock = ( diff --git a/cashu/mint/router.py b/cashu/mint/router.py index a719bf12..c1ebd306 100644 --- a/cashu/mint/router.py +++ b/cashu/mint/router.py @@ -1,5 +1,3 @@ -from typing import Any, Dict, List - from fastapi import APIRouter, Request, WebSocket from loguru import logger @@ -11,7 +9,6 @@ KeysetsResponseKeyset, KeysResponse, KeysResponseKeyset, - MintMeltMethodSetting, PostCheckStateRequest, PostCheckStateResponse, PostMeltQuoteRequest, @@ -44,43 +41,7 @@ ) async def info() -> GetInfoResponse: logger.trace("> GET /v1/info") - - # determine all method-unit pairs - method_settings: Dict[int, List[MintMeltMethodSetting]] = {} - for nut in [4, 5]: - method_settings[nut] = [] - for method, unit_dict in ledger.backends.items(): - for unit in unit_dict.keys(): - setting = MintMeltMethodSetting(method=method.name, unit=unit.name) - - if nut == 4 and settings.mint_max_peg_in: - setting.max_amount = settings.mint_max_peg_in - setting.min_amount = 0 - elif nut == 5 and settings.mint_max_peg_out: - setting.max_amount = settings.mint_max_peg_out - setting.min_amount = 0 - - method_settings[nut].append(setting) - - supported_dict = dict(supported=True) - - mint_features: Dict[int, Dict[str, Any]] = { - 4: dict( - methods=method_settings[4], - disabled=settings.mint_peg_out_only, - ), - 5: dict( - methods=method_settings[5], - disabled=False, - ), - 7: supported_dict, - 8: supported_dict, - 9: supported_dict, - 10: supported_dict, - 11: supported_dict, - 12: supported_dict, - } - + mint_features = ledger.mint_features() return GetInfoResponse( name=settings.mint_info_name, pubkey=ledger.pubkey.serialize().hex() if ledger.pubkey else None, diff --git a/cashu/wallet/cli/cli.py b/cashu/wallet/cli/cli.py index ef6b4ed4..d577078e 100644 --- a/cashu/wallet/cli/cli.py +++ b/cashu/wallet/cli/cli.py @@ -19,6 +19,7 @@ from ...core.helpers import sum_proofs from ...core.json_rpc.base import JSONRPCNotficationParams from ...core.logging import configure_logger +from ...core.nuts import WEBSOCKETS_NUT from ...core.settings import settings from ...nostr.client.client import NostrClient from ...tor.tor import TorProxy @@ -301,7 +302,7 @@ def mint_invoice_callback(msg: JSONRPCNotficationParams): # user requests an invoice if amount and not id: - mint_supports_websockets = True + mint_supports_websockets = wallet.mint_info.supports(WEBSOCKETS_NUT) invoice, subscription = await wallet.request_mint_subscription( amount, callback=mint_invoice_callback ) @@ -330,11 +331,10 @@ def mint_invoice_callback(msg: JSONRPCNotficationParams): # we still check manually every 10 seconds check_until = time.time() + 5 * 60 # check for five minutes - paid = False while time.time() < check_until and not paid: await asyncio.sleep(10) try: - # await wallet.mint(amount, split=optional_split, id=invoice.id) + await wallet.mint(amount, split=optional_split, id=invoice.id) paid = True except Exception as e: # TODO: user error codes! @@ -350,7 +350,7 @@ def mint_invoice_callback(msg: JSONRPCNotficationParams): " recheck after the invoice has been paid." ) - # user paid invoice and want to check it + # user paid invoice before and wants to check the quote id elif amount and id: await wallet.mint(amount, split=optional_split, id=id) From f4ffb8d6e5b0d8d85aafdf2e11615e5e64336731 Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Sun, 14 Apr 2024 11:16:12 +0200 Subject: [PATCH 11/45] startup fix --- cashu/mint/features.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/cashu/mint/features.py b/cashu/mint/features.py index 07fcf96d..22cf6d12 100644 --- a/cashu/mint/features.py +++ b/cashu/mint/features.py @@ -1,6 +1,6 @@ from typing import Any, Dict, List -from fastapi import APIRouter +from cashu.mint.protocols import SupportsBackends from ..core.base import ( MintMeltMethodSetting, @@ -17,18 +17,15 @@ WEBSOCKETS_NUT, ) from ..core.settings import settings -from ..mint.startup import ledger -router: APIRouter = APIRouter() - -class LedgerFeatures: +class LedgerFeatures(SupportsBackends): def mint_features(self) -> Dict[int, Dict[str, Any]]: # determine all method-unit pairs method_settings: Dict[int, List[MintMeltMethodSetting]] = {} for nut in [MINT_NUT, MELT_NUT]: method_settings[nut] = [] - for method, unit_dict in ledger.backends.items(): + for method, unit_dict in self.backends.items(): for unit in unit_dict.keys(): setting = MintMeltMethodSetting(method=method.name, unit=unit.name) From cf52224bf670ac155c5949475474b2d318a54ad0 Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Sun, 14 Apr 2024 11:18:38 +0200 Subject: [PATCH 12/45] works with old mints --- cashu/wallet/cli/cli.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/cashu/wallet/cli/cli.py b/cashu/wallet/cli/cli.py index d577078e..fa6817d5 100644 --- a/cashu/wallet/cli/cli.py +++ b/cashu/wallet/cli/cli.py @@ -303,10 +303,13 @@ def mint_invoice_callback(msg: JSONRPCNotficationParams): # user requests an invoice if amount and not id: mint_supports_websockets = wallet.mint_info.supports(WEBSOCKETS_NUT) - invoice, subscription = await wallet.request_mint_subscription( - amount, callback=mint_invoice_callback - ) - invoice_nonlocal, subscription_nonlocal = invoice, subscription + if mint_supports_websockets: + invoice, subscription = await wallet.request_mint_subscription( + amount, callback=mint_invoice_callback + ) + invoice_nonlocal, subscription_nonlocal = invoice, subscription + else: + invoice = await wallet.request_mint(amount) if invoice.bolt11: print("") print(f"Pay invoice to mint {wallet.unit.str(amount)}:") @@ -332,7 +335,7 @@ def mint_invoice_callback(msg: JSONRPCNotficationParams): # we still check manually every 10 seconds check_until = time.time() + 5 * 60 # check for five minutes while time.time() < check_until and not paid: - await asyncio.sleep(10) + await asyncio.sleep(5) try: await wallet.mint(amount, split=optional_split, id=invoice.id) paid = True From 5a5eaae63d430d5dbbc9cf2d3240eaee79a5b118 Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Sun, 14 Apr 2024 18:25:59 +0200 Subject: [PATCH 13/45] wip cli --- cashu/core/base.py | 2 +- cashu/mint/tasks.py | 4 ++++ cashu/wallet/cli/cli.py | 4 ++-- cashu/wallet/subscriptions.py | 1 - tests/test_wallet_cli.py | 2 +- 5 files changed, 8 insertions(+), 5 deletions(-) diff --git a/cashu/core/base.py b/cashu/core/base.py index 5c4e63a9..feafd70d 100644 --- a/cashu/core/base.py +++ b/cashu/core/base.py @@ -534,7 +534,7 @@ class ProofState(LedgerEvent): state: SpentState witness: Optional[str] = None - @root_validator(pre=True) + @root_validator() def check_witness(cls, values): state, witness = values.get("state"), values.get("witness") if witness is not None and state != SpentState.spent: diff --git a/cashu/mint/tasks.py b/cashu/mint/tasks.py index 56cfc1d6..9e88900b 100644 --- a/cashu/mint/tasks.py +++ b/cashu/mint/tasks.py @@ -35,4 +35,8 @@ async def invoice_callback_dispatcher(self, checking_id: str) -> None: if not quote: logger.error(f"Quote not found for {checking_id}") return + # set the quote as paid + if not quote.paid: + quote.paid = True + await self.crud.update_mint_quote(quote=quote, db=self.db) await self.events.submit(quote) diff --git a/cashu/wallet/cli/cli.py b/cashu/wallet/cli/cli.py index fa6817d5..56d529ba 100644 --- a/cashu/wallet/cli/cli.py +++ b/cashu/wallet/cli/cli.py @@ -294,11 +294,11 @@ def mint_invoice_callback(msg: JSONRPCNotficationParams): asyncio.run( wallet.mint(int(amount), split=optional_split, id=invoice.id) ) + # set paid so we won't react to any more callbacks + paid = True except Exception as e: print(f"Error during mint: {str(e)}") return - # set paid so we won't react to any more callbacks - paid = True # user requests an invoice if amount and not id: diff --git a/cashu/wallet/subscriptions.py b/cashu/wallet/subscriptions.py index 2a132cef..de6aa006 100644 --- a/cashu/wallet/subscriptions.py +++ b/cashu/wallet/subscriptions.py @@ -36,7 +36,6 @@ def __init__(self, url: str): def _on_message(self, ws, message): try: msg = JSONRPCResponse.parse_raw(message) - print(msg) return except Exception: pass diff --git a/tests/test_wallet_cli.py b/tests/test_wallet_cli.py index 59fc5176..7e58f4bd 100644 --- a/tests/test_wallet_cli.py +++ b/tests/test_wallet_cli.py @@ -117,7 +117,7 @@ def test_invoice_automatic_fakewallet(mint, cli_prefix): [*cli_prefix, "invoice", "1000"], ) assert result.exception is None - print("INVOICE") + print("INVOICE AUTOMATIC") print(result.output) wallet = asyncio.run(init_wallet()) assert wallet.available_balance >= 1000 From c2dba4d4c6fdd86d8483befde1f3d77a7f5733d4 Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Mon, 15 Apr 2024 00:38:02 +0200 Subject: [PATCH 14/45] fix mypy --- cashu/mint/events/client_manager.py | 15 ++++++++------- cashu/wallet/subscriptions.py | 3 ++- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/cashu/mint/events/client_manager.py b/cashu/mint/events/client_manager.py index 0dd71ad6..9adb0abe 100644 --- a/cashu/mint/events/client_manager.py +++ b/cashu/mint/events/client_manager.py @@ -75,7 +75,8 @@ async def start(self): await self._send_msg(resp) continue try: - resp = await self._handle_request(req) + # handle the request + await self._handle_request(req) except Exception as e: resp = JSONRPCErrorResponse( error=JSONRPCError( @@ -89,19 +90,19 @@ async def start(self): async def _handle_request(self, data: JSONRPCRequest) -> JSONRPCResponse: logger.info(f"Received message: {data}") if data.method == JSONRPCMethods.SUBSCRIBE.value: - params = JSONRPCSubscribeParams.parse_obj(data.params) - self.add_subscription(params.filters, params.subId) + subscribe_params = JSONRPCSubscribeParams.parse_obj(data.params) + self.add_subscription(subscribe_params.filters, subscribe_params.subId) result = JSONRRPCSubscribeResponse( status=JSONRPCStatus.OK, - subId=params.subId, + subId=subscribe_params.subId, ) return JSONRPCResponse(result=result.dict(), id=data.id) elif data.method == JSONRPCMethods.UNSUBSCRIBE.value: - params = JSONRPCUnubscribeParams.parse_obj(data.params) - self.remove_subscription(params.subId) + unsubscribe_params = JSONRPCUnubscribeParams.parse_obj(data.params) + self.remove_subscription(unsubscribe_params.subId) result = JSONRRPCSubscribeResponse( status=JSONRPCStatus.OK, - subId=params.subId, + subId=unsubscribe_params.subId, ) return JSONRPCResponse(result=result.dict(), id=data.id) else: diff --git a/cashu/wallet/subscriptions.py b/cashu/wallet/subscriptions.py index de6aa006..eae85206 100644 --- a/cashu/wallet/subscriptions.py +++ b/cashu/wallet/subscriptions.py @@ -35,7 +35,8 @@ def __init__(self, url: str): def _on_message(self, ws, message): try: - msg = JSONRPCResponse.parse_raw(message) + # return if message is a response + JSONRPCResponse.parse_raw(message) return except Exception: pass From 1853e90839039a3f540d9a22ce9ff0190c9797be Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Mon, 15 Apr 2024 00:56:30 +0200 Subject: [PATCH 15/45] remove automatic invoice test now with websockets --- tests/test_wallet_cli.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/test_wallet_cli.py b/tests/test_wallet_cli.py index 7e58f4bd..765e2cda 100644 --- a/tests/test_wallet_cli.py +++ b/tests/test_wallet_cli.py @@ -109,20 +109,20 @@ def test_balance(cli_prefix): assert result.exit_code == 0 -@pytest.mark.skipif(not is_fake, reason="only on fakewallet") -def test_invoice_automatic_fakewallet(mint, cli_prefix): - runner = CliRunner() - result = runner.invoke( - cli, - [*cli_prefix, "invoice", "1000"], - ) - assert result.exception is None - print("INVOICE AUTOMATIC") - print(result.output) - wallet = asyncio.run(init_wallet()) - assert wallet.available_balance >= 1000 - assert f"Balance: {wallet.available_balance} sat" in result.output - assert result.exit_code == 0 +# @pytest.mark.skipif(not is_fake, reason="only on fakewallet") +# def test_invoice_automatic_fakewallet(mint, cli_prefix): +# runner = CliRunner() +# result = runner.invoke( +# cli, +# [*cli_prefix, "invoice", "1000"], +# ) +# assert result.exception is None +# print("INVOICE AUTOMATIC") +# print(result.output) +# wallet = asyncio.run(init_wallet()) +# assert wallet.available_balance >= 1000 +# assert f"Balance: {wallet.available_balance} sat" in result.output +# assert result.exit_code == 0 def test_invoice(mint, cli_prefix): From b1f05c174c4e503ceaaa815e576e8fe51c991633 Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Mon, 15 Apr 2024 14:12:05 +0200 Subject: [PATCH 16/45] remove comment --- tests/test_wallet_cli.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/tests/test_wallet_cli.py b/tests/test_wallet_cli.py index 765e2cda..26dfa421 100644 --- a/tests/test_wallet_cli.py +++ b/tests/test_wallet_cli.py @@ -109,22 +109,6 @@ def test_balance(cli_prefix): assert result.exit_code == 0 -# @pytest.mark.skipif(not is_fake, reason="only on fakewallet") -# def test_invoice_automatic_fakewallet(mint, cli_prefix): -# runner = CliRunner() -# result = runner.invoke( -# cli, -# [*cli_prefix, "invoice", "1000"], -# ) -# assert result.exception is None -# print("INVOICE AUTOMATIC") -# print(result.output) -# wallet = asyncio.run(init_wallet()) -# assert wallet.available_balance >= 1000 -# assert f"Balance: {wallet.available_balance} sat" in result.output -# assert result.exit_code == 0 - - def test_invoice(mint, cli_prefix): runner = CliRunner() result = runner.invoke( From fd58cc2ef6327c578186629acc15272dcdb4fb5b Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Tue, 16 Apr 2024 09:27:28 +0200 Subject: [PATCH 17/45] better logging --- cashu/lightning/fake.py | 2 +- cashu/mint/events/client_manager.py | 4 +++- cashu/mint/events/events.py | 5 +++++ cashu/mint/tasks.py | 3 +++ cashu/wallet/subscriptions.py | 1 + 5 files changed, 13 insertions(+), 2 deletions(-) diff --git a/cashu/lightning/fake.py b/cashu/lightning/fake.py index a6ce7248..bdce3748 100644 --- a/cashu/lightning/fake.py +++ b/cashu/lightning/fake.py @@ -55,7 +55,7 @@ async def status(self) -> StatusResponse: return StatusResponse(error_message=None, balance=1337) async def mark_invoice_paid(self, invoice: Bolt11) -> None: - await asyncio.sleep(2) + await asyncio.sleep(10) self.paid_invoices_incoming.append(invoice) await self.paid_invoices_queue.put(invoice) diff --git a/cashu/mint/events/client_manager.py b/cashu/mint/events/client_manager.py index 9adb0abe..1b786ea2 100644 --- a/cashu/mint/events/client_manager.py +++ b/cashu/mint/events/client_manager.py @@ -76,7 +76,9 @@ async def start(self): continue try: # handle the request - await self._handle_request(req) + logger.debug(f"Request: {req}") + handle_respo = await self._handle_request(req) + logger.debug(f"Response: {handle_respo}") except Exception as e: resp = JSONRPCErrorResponse( error=JSONRPCError( diff --git a/cashu/mint/events/events.py b/cashu/mint/events/events.py index e0656fb9..fe533837 100644 --- a/cashu/mint/events/events.py +++ b/cashu/mint/events/events.py @@ -1,5 +1,6 @@ from abc import ABC, abstractmethod +from loguru import logger from pydantic import BaseModel from .client_manager import LedgerEventClientManager @@ -32,6 +33,7 @@ def add_client(self, client: LedgerEventClientManager) -> bool: if len(self.clients) >= self.MAX_CLIENTS: return False self.clients.append(client) + logger.success(f"Added websocket subscription client {client}") return True def remove_client(self, client: LedgerEventClientManager) -> None: @@ -46,4 +48,7 @@ async def submit(self, event: LedgerEvent) -> None: for client in self.clients: for sub in client.subscriptions.get(event.identifier, []): + logger.trace( + f"Submitting event to sub {sub}: {self.serialize_event(event)}" + ) await client._send_obj(self.serialize_event(event), subId=sub) diff --git a/cashu/mint/tasks.py b/cashu/mint/tasks.py index 9e88900b..bfa49bb8 100644 --- a/cashu/mint/tasks.py +++ b/cashu/mint/tasks.py @@ -31,6 +31,8 @@ async def invoice_listener(self, backend: LightningBackend) -> None: async def invoice_callback_dispatcher(self, checking_id: str) -> None: logger.success(f"invoice callback dispatcher: {checking_id}") + # TODO: Explicitly check for the quote payment state before setting it as paid + # db read, quote.paid = True, db write should be refactored and moved to ledger.py quote = await self.crud.get_mint_quote(checking_id=checking_id, db=self.db) if not quote: logger.error(f"Quote not found for {checking_id}") @@ -39,4 +41,5 @@ async def invoice_callback_dispatcher(self, checking_id: str) -> None: if not quote.paid: quote.paid = True await self.crud.update_mint_quote(quote=quote, db=self.db) + logger.trace(f"Quote {quote} set as paid and ") await self.events.submit(quote) diff --git a/cashu/wallet/subscriptions.py b/cashu/wallet/subscriptions.py index eae85206..d5782355 100644 --- a/cashu/wallet/subscriptions.py +++ b/cashu/wallet/subscriptions.py @@ -34,6 +34,7 @@ def __init__(self, url: str): self.websocket = WebSocketApp(ws_url, on_message=self._on_message) def _on_message(self, ws, message): + logger.trace(f"Received message: {message}") try: # return if message is a response JSONRPCResponse.parse_raw(message) From 0aba3d337484b6d8dbf5f490dd13df503d011e53 Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Tue, 16 Apr 2024 18:19:50 +0200 Subject: [PATCH 18/45] send back response --- cashu/mint/events/client_manager.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cashu/mint/events/client_manager.py b/cashu/mint/events/client_manager.py index 1b786ea2..06102794 100644 --- a/cashu/mint/events/client_manager.py +++ b/cashu/mint/events/client_manager.py @@ -77,8 +77,7 @@ async def start(self): try: # handle the request logger.debug(f"Request: {req}") - handle_respo = await self._handle_request(req) - logger.debug(f"Response: {handle_respo}") + resp = await self._handle_request(req) except Exception as e: resp = JSONRPCErrorResponse( error=JSONRPCError( From 8884dc80e784b8b43b0b871e6e9c3c7ca3dbd8ee Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Tue, 16 Apr 2024 19:04:07 +0200 Subject: [PATCH 19/45] add rate limiter to websocket --- cashu/core/json_rpc/base.py | 2 +- cashu/lightning/fake.py | 2 +- cashu/mint/events/client_manager.py | 18 +++++++++++++----- cashu/mint/events/events.py | 2 +- cashu/mint/router.py | 8 ++++++++ cashu/wallet/subscriptions.py | 11 +++++++++++ 6 files changed, 35 insertions(+), 8 deletions(-) diff --git a/cashu/core/json_rpc/base.py b/cashu/core/json_rpc/base.py index 87514986..fb8ba52f 100644 --- a/cashu/core/json_rpc/base.py +++ b/cashu/core/json_rpc/base.py @@ -65,7 +65,7 @@ class JSONRPCSubscribeParams(BaseModel): subId: str -class JSONRPCUnubscribeParams(BaseModel): +class JSONRPCUnsubscribeParams(BaseModel): subId: str diff --git a/cashu/lightning/fake.py b/cashu/lightning/fake.py index bdce3748..5cd85ab3 100644 --- a/cashu/lightning/fake.py +++ b/cashu/lightning/fake.py @@ -55,7 +55,7 @@ async def status(self) -> StatusResponse: return StatusResponse(error_message=None, balance=1337) async def mark_invoice_paid(self, invoice: Bolt11) -> None: - await asyncio.sleep(10) + await asyncio.sleep(5) self.paid_invoices_incoming.append(invoice) await self.paid_invoices_queue.put(invoice) diff --git a/cashu/mint/events/client_manager.py b/cashu/mint/events/client_manager.py index 06102794..75de493e 100644 --- a/cashu/mint/events/client_manager.py +++ b/cashu/mint/events/client_manager.py @@ -15,24 +15,26 @@ JSONRPCResponse, JSONRPCStatus, JSONRPCSubscribeParams, - JSONRPCUnubscribeParams, + JSONRPCUnsubscribeParams, JSONRRPCSubscribeResponse, ) class LedgerEventClientManager: websocket: WebSocket - subscriptions: dict[str, List[str]] = {} # filter -> List[subId] + subscriptions: dict[str, List[str]] = {} # [filter, List[subId]] max_subscriptions = 100 def __init__(self, websocket: WebSocket): self.websocket = websocket + self.subscriptions = {} async def start(self): await self.websocket.accept() while True: json_data = await self.websocket.receive_text() + # Parse the JSON data try: data = json.loads(json_data) except json.JSONDecodeError as e: @@ -47,6 +49,7 @@ async def start(self): await self._send_msg(resp) continue + # Parse the JSONRPCRequest try: req = JSONRPCRequest.parse_obj(data) except Exception as e: @@ -61,7 +64,7 @@ async def start(self): logger.warning(f"Error handling websocket message: {e}") continue - # check if method is in the enum + # Check if the method is valid try: JSONRPCMethods(req.method) except ValueError: @@ -74,8 +77,9 @@ async def start(self): ) await self._send_msg(resp) continue + + # Handle the request try: - # handle the request logger.debug(f"Request: {req}") resp = await self._handle_request(req) except Exception as e: @@ -86,6 +90,8 @@ async def start(self): ), id=req.id, ) + + # Send the response await self._send_msg(resp) async def _handle_request(self, data: JSONRPCRequest) -> JSONRPCResponse: @@ -99,7 +105,7 @@ async def _handle_request(self, data: JSONRPCRequest) -> JSONRPCResponse: ) return JSONRPCResponse(result=result.dict(), id=data.id) elif data.method == JSONRPCMethods.UNSUBSCRIBE.value: - unsubscribe_params = JSONRPCUnubscribeParams.parse_obj(data.params) + unsubscribe_params = JSONRPCUnsubscribeParams.parse_obj(data.params) self.remove_subscription(unsubscribe_params.subId) result = JSONRRPCSubscribeResponse( status=JSONRPCStatus.OK, @@ -129,12 +135,14 @@ def add_subscription(self, filters: List[str], subId: str) -> None: for filter in filters: if filter not in self.subscriptions: self.subscriptions[filter] = [] + logger.debug(f"Adding subscription {subId} for filter {filter}") self.subscriptions[filter].append(subId) def remove_subscription(self, subId: str) -> None: for filter, subs in self.subscriptions.items(): for sub in subs: if sub == subId: + logger.debug(f"Removing subscription {subId} for filter {filter}") self.subscriptions[filter].remove(sub) return raise ValueError(f"Subscription not found: {subId}") diff --git a/cashu/mint/events/events.py b/cashu/mint/events/events.py index fe533837..01c12d15 100644 --- a/cashu/mint/events/events.py +++ b/cashu/mint/events/events.py @@ -33,7 +33,7 @@ def add_client(self, client: LedgerEventClientManager) -> bool: if len(self.clients) >= self.MAX_CLIENTS: return False self.clients.append(client) - logger.success(f"Added websocket subscription client {client}") + logger.debug(f"Added websocket subscription client {client}") return True def remove_client(self, client: LedgerEventClientManager) -> None: diff --git a/cashu/mint/router.py b/cashu/mint/router.py index c1ebd306..09f4dfc3 100644 --- a/cashu/mint/router.py +++ b/cashu/mint/router.py @@ -1,4 +1,5 @@ from fastapi import APIRouter, Request, WebSocket +from limits import RateLimitItemPerMinute from loguru import logger from cashu.mint.events.client_manager import LedgerEventClientManager @@ -189,6 +190,13 @@ async def get_mint_quote(request: Request, quote: str) -> PostMintQuoteResponse: @router.websocket("/v1/ws", name="Websocket endpoint for subscriptions") async def websocket_endpoint(websocket: WebSocket): + success = limiter._limiter.hit( + RateLimitItemPerMinute(settings.mint_transaction_rate_limit_per_minute), + websocket.client.host if websocket.client else "unknown", + ) + if not success: + await websocket.close(code=1008, reason="Rate limit exceeded") + return client = LedgerEventClientManager(websocket=websocket) success = ledger.events.add_client(client) if not success: diff --git a/cashu/wallet/subscriptions.py b/cashu/wallet/subscriptions.py index d5782355..970477f0 100644 --- a/cashu/wallet/subscriptions.py +++ b/cashu/wallet/subscriptions.py @@ -12,6 +12,7 @@ JSONRPCRequest, JSONRPCResponse, JSONRPCSubscribeParams, + JSONRPCUnsubscribeParams, ) @@ -57,6 +58,16 @@ def connect(self): self.websocket.run_forever(ping_interval=10, ping_timeout=5) def close(self): + # unsubscribe from all subscriptions + for subId in self.callback_map.keys(): + req = JSONRPCRequest( + method="unsubscribe", + params=JSONRPCUnsubscribeParams(subId=subId).dict(), + id=self.id_counter, + ) + self.websocket.send(req.json()) + self.id_counter += 1 + self.websocket.keep_running = False self.websocket.close() From 11ef3e5a337a40abd48ac5f5ef67b7bb57922736 Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Tue, 16 Apr 2024 19:13:43 +0200 Subject: [PATCH 20/45] add rate limiter to subscriptions --- cashu/mint/events/client_manager.py | 19 +++++++++++++++++++ cashu/mint/limit.py | 14 ++++++++++++++ cashu/mint/router.py | 11 ++--------- 3 files changed, 35 insertions(+), 9 deletions(-) diff --git a/cashu/mint/events/client_manager.py b/cashu/mint/events/client_manager.py index 75de493e..35f9cc0a 100644 --- a/cashu/mint/events/client_manager.py +++ b/cashu/mint/events/client_manager.py @@ -4,6 +4,8 @@ from fastapi import WebSocket from loguru import logger +from cashu.mint.limit import assert_limit + from ...core.json_rpc.base import ( JSONRPCError, JSONRPCErrorCode, @@ -34,6 +36,23 @@ async def start(self): while True: json_data = await self.websocket.receive_text() + # Check the rate limit + try: + assert_limit( + self.websocket.client.host if self.websocket.client else "unknown" + ) + except Exception as e: + logger.error(f"Error: {e}") + resp = JSONRPCErrorResponse( + error=JSONRPCError( + code=JSONRPCErrorCode.SERVER_ERROR, + message=f"Error: {e}", + ), + id=0, + ) + await self._send_msg(resp) + continue + # Parse the JSON data try: data = json.loads(json_data) diff --git a/cashu/mint/limit.py b/cashu/mint/limit.py index 1a8a4c28..50b0fcbe 100644 --- a/cashu/mint/limit.py +++ b/cashu/mint/limit.py @@ -1,5 +1,6 @@ from fastapi import status from fastapi.responses import JSONResponse +from limits import RateLimitItemPerMinute from loguru import logger from slowapi import Limiter from slowapi.util import get_remote_address @@ -39,3 +40,16 @@ def get_remote_address_excluding_local(request: Request) -> str: default_limits=[f"{settings.mint_transaction_rate_limit_per_minute}/minute"], enabled=settings.mint_rate_limit, ) + + +def assert_limit(identifier: str): + global limiter + success = limiter._limiter.hit( + RateLimitItemPerMinute(settings.mint_transaction_rate_limit_per_minute), + identifier, + ) + if not success: + logger.warning( + f"Rate limit {settings.mint_transaction_rate_limit_per_minute}/minute exceeded: {identifier}" + ) + raise Exception("Rate limit exceeded") diff --git a/cashu/mint/router.py b/cashu/mint/router.py index 09f4dfc3..0b225979 100644 --- a/cashu/mint/router.py +++ b/cashu/mint/router.py @@ -1,5 +1,4 @@ from fastapi import APIRouter, Request, WebSocket -from limits import RateLimitItemPerMinute from loguru import logger from cashu.mint.events.client_manager import LedgerEventClientManager @@ -28,7 +27,7 @@ from ..core.errors import KeysetNotFoundError from ..core.settings import settings from ..mint.startup import ledger -from .limit import limiter +from .limit import assert_limit, limiter router: APIRouter = APIRouter() @@ -190,13 +189,7 @@ async def get_mint_quote(request: Request, quote: str) -> PostMintQuoteResponse: @router.websocket("/v1/ws", name="Websocket endpoint for subscriptions") async def websocket_endpoint(websocket: WebSocket): - success = limiter._limiter.hit( - RateLimitItemPerMinute(settings.mint_transaction_rate_limit_per_minute), - websocket.client.host if websocket.client else "unknown", - ) - if not success: - await websocket.close(code=1008, reason="Rate limit exceeded") - return + assert_limit(websocket.client.host if websocket.client else "unknown") client = LedgerEventClientManager(websocket=websocket) success = ledger.events.add_client(client) if not success: From bf71c7e5da66f2966a627ec05caad2fff1921fc3 Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Tue, 16 Apr 2024 23:58:23 +0200 Subject: [PATCH 21/45] refactor websocket ratelimit --- cashu/mint/events/client_manager.py | 6 ++-- cashu/mint/limit.py | 43 ++++++++++++++++++++++++++++- cashu/mint/router.py | 4 +-- 3 files changed, 46 insertions(+), 7 deletions(-) diff --git a/cashu/mint/events/client_manager.py b/cashu/mint/events/client_manager.py index 35f9cc0a..9d8e347b 100644 --- a/cashu/mint/events/client_manager.py +++ b/cashu/mint/events/client_manager.py @@ -4,7 +4,7 @@ from fastapi import WebSocket from loguru import logger -from cashu.mint.limit import assert_limit +from cashu.mint.limit import limit_websocket from ...core.json_rpc.base import ( JSONRPCError, @@ -38,9 +38,7 @@ async def start(self): # Check the rate limit try: - assert_limit( - self.websocket.client.host if self.websocket.client else "unknown" - ) + limit_websocket(self.websocket) except Exception as e: logger.error(f"Error: {e}") resp = JSONRPCErrorResponse( diff --git a/cashu/mint/limit.py b/cashu/mint/limit.py index 50b0fcbe..b0840ff0 100644 --- a/cashu/mint/limit.py +++ b/cashu/mint/limit.py @@ -1,4 +1,4 @@ -from fastapi import status +from fastapi import WebSocket, status from fastapi.responses import JSONResponse from limits import RateLimitItemPerMinute from loguru import logger @@ -43,6 +43,16 @@ def get_remote_address_excluding_local(request: Request) -> str: def assert_limit(identifier: str): + """Custom rate limit handler that accepts a string identifier + and raises an exception if the rate limit is exceeded. Uses the + setting `mint_transaction_rate_limit_per_minute` for the rate limit. + + Args: + identifier (str): The identifier to use for the rate limit. IP address for example. + + Raises: + Exception: If the rate limit is exceeded. + """ global limiter success = limiter._limiter.hit( RateLimitItemPerMinute(settings.mint_transaction_rate_limit_per_minute), @@ -53,3 +63,34 @@ def assert_limit(identifier: str): f"Rate limit {settings.mint_transaction_rate_limit_per_minute}/minute exceeded: {identifier}" ) raise Exception("Rate limit exceeded") + + +def get_ws_remote_address(ws: WebSocket) -> str: + """Returns the ip address for the current websocket (or 127.0.0.1 if none found) + + Args: + ws (WebSocket): The FastAPI WebSocket object. + + Returns: + str: The ip address for the current websocket. + """ + if not ws.client or not ws.client.host: + return "127.0.0.1" + + return ws.client.host + + +def limit_websocket(ws: WebSocket): + """Websocket rate limit handler that accepts a FastAPI WebSocket object. + This function will raise an exception if the rate limit is exceeded. + + Args: + ws (WebSocket): The FastAPI WebSocket object. + + Raises: + Exception: If the rate limit is exceeded. + """ + remote_address = get_ws_remote_address(ws) + if remote_address == "127.0.0.1": + return + assert_limit(remote_address) diff --git a/cashu/mint/router.py b/cashu/mint/router.py index 0b225979..c29fcb11 100644 --- a/cashu/mint/router.py +++ b/cashu/mint/router.py @@ -27,7 +27,7 @@ from ..core.errors import KeysetNotFoundError from ..core.settings import settings from ..mint.startup import ledger -from .limit import assert_limit, limiter +from .limit import limit_websocket, limiter router: APIRouter = APIRouter() @@ -189,7 +189,7 @@ async def get_mint_quote(request: Request, quote: str) -> PostMintQuoteResponse: @router.websocket("/v1/ws", name="Websocket endpoint for subscriptions") async def websocket_endpoint(websocket: WebSocket): - assert_limit(websocket.client.host if websocket.client else "unknown") + limit_websocket(websocket) client = LedgerEventClientManager(websocket=websocket) success = ledger.events.add_client(client) if not success: From 1368de5dc681020d1815d72e49290aa469d0ccdf Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Wed, 17 Apr 2024 08:58:58 +0200 Subject: [PATCH 22/45] websocket tests --- cashu/core/settings.py | 3 +- cashu/lightning/fake.py | 7 +++-- cashu/mint/router.py | 3 +- tests/conftest.py | 4 ++- tests/test_wallet_subscriptions.py | 49 ++++++++++++++++++++++++++++++ 5 files changed, 60 insertions(+), 6 deletions(-) create mode 100644 tests/test_wallet_subscriptions.py diff --git a/cashu/core/settings.py b/cashu/core/settings.py index 4be3ccea..5134587e 100644 --- a/cashu/core/settings.py +++ b/cashu/core/settings.py @@ -115,7 +115,8 @@ class MintLimits(MintSettings): class FakeWalletSettings(MintSettings): fakewallet_brr: bool = Field(default=True) - fakewallet_delay_payment: bool = Field(default=False) + fakewallet_delay_outgoing_payment: Optional[int] = Field(default=3) + fakewallet_delay_incoming_payment: Optional[int] = Field(default=3) fakewallet_stochastic_invoice: bool = Field(default=False) fakewallet_payment_state: Optional[bool] = Field(default=None) mint_cache_secrets: bool = Field(default=True) diff --git a/cashu/lightning/fake.py b/cashu/lightning/fake.py index 5cd85ab3..55bd2d6b 100644 --- a/cashu/lightning/fake.py +++ b/cashu/lightning/fake.py @@ -55,7 +55,8 @@ async def status(self) -> StatusResponse: return StatusResponse(error_message=None, balance=1337) async def mark_invoice_paid(self, invoice: Bolt11) -> None: - await asyncio.sleep(5) + if settings.fakewallet_delay_incoming_payment: + await asyncio.sleep(settings.fakewallet_delay_incoming_payment) self.paid_invoices_incoming.append(invoice) await self.paid_invoices_queue.put(invoice) @@ -140,8 +141,8 @@ async def create_invoice( async def pay_invoice(self, quote: MeltQuote, fee_limit: int) -> PaymentResponse: invoice = decode(quote.request) - if settings.fakewallet_delay_payment: - await asyncio.sleep(5) + if settings.fakewallet_delay_outgoing_payment: + await asyncio.sleep(settings.fakewallet_delay_outgoing_payment) if invoice.payment_hash in self.payment_secrets or settings.fakewallet_brr: if invoice not in self.paid_invoices_outgoing: diff --git a/cashu/mint/router.py b/cashu/mint/router.py index c29fcb11..f9250fbe 100644 --- a/cashu/mint/router.py +++ b/cashu/mint/router.py @@ -197,7 +197,8 @@ async def websocket_endpoint(websocket: WebSocket): return try: await client.start() - except Exception: + except Exception as e: + logger.debug(f"Exception: {e}") ledger.events.remove_client(client) diff --git a/tests/conftest.py b/tests/conftest.py index e6f31cab..a6240d54 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -33,7 +33,8 @@ settings.wallet_unit = "sat" settings.mint_backend_bolt11_sat = settings.mint_backend_bolt11_sat or "FakeWallet" settings.fakewallet_brr = True -settings.fakewallet_delay_payment = False +settings.fakewallet_delay_outgoing_payment = None +settings.fakewallet_delay_incoming_payment = 2 settings.fakewallet_stochastic_invoice = False assert ( settings.mint_test_database != settings.mint_database @@ -44,6 +45,7 @@ settings.mint_private_key = "TEST_PRIVATE_KEY" settings.mint_seed_decryption_key = "" settings.mint_max_balance = 0 +settings.mint_transaction_rate_limit_per_minute = 60 assert "test" in settings.cashu_dir shutil.rmtree(settings.cashu_dir, ignore_errors=True) diff --git a/tests/test_wallet_subscriptions.py b/tests/test_wallet_subscriptions.py new file mode 100644 index 00000000..757a371c --- /dev/null +++ b/tests/test_wallet_subscriptions.py @@ -0,0 +1,49 @@ +import asyncio + +import pytest +import pytest_asyncio + +from cashu.core.json_rpc.base import JSONRPCNotficationParams +from cashu.core.nuts import WEBSOCKETS_NUT +from cashu.core.settings import settings +from cashu.wallet.wallet import Wallet +from tests.conftest import SERVER_ENDPOINT +from tests.helpers import ( + pay_if_regtest, +) + + +@pytest_asyncio.fixture(scope="function") +async def wallet(mint): + wallet1 = await Wallet.with_db( + url=SERVER_ENDPOINT, + db="test_data/wallet_subscriptions", + name="wallet_subscriptions", + ) + await wallet1.load_mint() + yield wallet1 + + +@pytest.mark.asyncio +async def test_wallet_subscription(wallet: Wallet): + assert wallet.mint_info.supports(WEBSOCKETS_NUT) + triggered = False + msg_stack: list[JSONRPCNotficationParams] = [] + + def callback(msg: JSONRPCNotficationParams): + nonlocal triggered, msg_stack + triggered = True + msg_stack.append(msg) + asyncio.run(wallet.mint(int(invoice.amount), id=invoice.id)) + + invoice, sub = await wallet.request_mint_subscription(128, callback=callback) + pay_if_regtest(invoice.bolt11) + await asyncio.sleep(settings.fakewallet_delay_incoming_payment or 3) + assert triggered + assert len(msg_stack) == 2 + + assert msg_stack[0].payload["paid"] is True + assert msg_stack[0].payload["issued"] is False + + assert msg_stack[1].payload["paid"] is True + assert msg_stack[1].payload["issued"] is True From 3a6b68784543f2e6872e960e8204496cb484cce4 Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Wed, 17 Apr 2024 10:48:54 +0200 Subject: [PATCH 23/45] subscription kinds --- cashu/core/base.py | 14 +++++++++ cashu/core/json_rpc/base.py | 11 +++++-- cashu/mint/events/client_manager.py | 47 ++++++++++++++++++++--------- cashu/mint/events/events.py | 9 +++++- cashu/wallet/subscriptions.py | 14 ++++++--- cashu/wallet/wallet.py | 8 ++++- tests/test_wallet_subscriptions.py | 3 +- 7 files changed, 83 insertions(+), 23 deletions(-) diff --git a/cashu/core/base.py b/cashu/core/base.py index 368bca24..6dc5097c 100644 --- a/cashu/core/base.py +++ b/cashu/core/base.py @@ -9,6 +9,8 @@ from loguru import logger from pydantic import BaseModel, Field, root_validator +from cashu.core.json_rpc.base import JSONRPCSubscriptionKinds + from ..mint.events.events import LedgerEvent from .crypto.aes import AESCipher from .crypto.b_dhke import hash_to_curve @@ -260,6 +262,10 @@ def identifier(self) -> str: """Implementation of the abstract method from LedgerEventManager""" return self.quote + @property + def kind(self) -> JSONRPCSubscriptionKinds: + return JSONRPCSubscriptionKinds.BOLT11_MELT_QUOTE + class MintQuote(LedgerEvent): quote: str @@ -304,6 +310,10 @@ def identifier(self) -> str: """Implementation of the abstract method from LedgerEventManager""" return self.quote + @property + def kind(self) -> JSONRPCSubscriptionKinds: + return JSONRPCSubscriptionKinds.BOLT11_MINT_QUOTE + # ------- API ------- @@ -546,6 +556,10 @@ def identifier(self) -> str: """Implementation of the abstract method from LedgerEventManager""" return self.Y + @property + def kind(self) -> JSONRPCSubscriptionKinds: + return JSONRPCSubscriptionKinds.PROOF_STATE + class PostCheckStateResponse(BaseModel): states: List[ProofState] = [] diff --git a/cashu/core/json_rpc/base.py b/cashu/core/json_rpc/base.py index fb8ba52f..2b7c7fe2 100644 --- a/cashu/core/json_rpc/base.py +++ b/cashu/core/json_rpc/base.py @@ -52,8 +52,14 @@ class JSONRPCErrorResponse(BaseModel): class JSONRPCMethods(Enum): - SUBSCRIBE = "subscribe" - UNSUBSCRIBE = "unsubscribe" + SUBSCRIBE = "sub" + UNSUBSCRIBE = "unsub" + + +class JSONRPCSubscriptionKinds(Enum): + BOLT11_MINT_QUOTE = "bolt11_mint_quote" + BOLT11_MELT_QUOTE = "bolt11_melt_quote" + PROOF_STATE = "proof_state" class JSONRPCStatus(Enum): @@ -61,6 +67,7 @@ class JSONRPCStatus(Enum): class JSONRPCSubscribeParams(BaseModel): + kind: JSONRPCSubscriptionKinds filters: List[str] = Field(..., max_length=settings.mint_max_request_length) subId: str diff --git a/cashu/mint/events/client_manager.py b/cashu/mint/events/client_manager.py index 9d8e347b..1c89be92 100644 --- a/cashu/mint/events/client_manager.py +++ b/cashu/mint/events/client_manager.py @@ -4,8 +4,6 @@ from fastapi import WebSocket from loguru import logger -from cashu.mint.limit import limit_websocket - from ...core.json_rpc.base import ( JSONRPCError, JSONRPCErrorCode, @@ -17,14 +15,18 @@ JSONRPCResponse, JSONRPCStatus, JSONRPCSubscribeParams, + JSONRPCSubscriptionKinds, JSONRPCUnsubscribeParams, JSONRRPCSubscribeResponse, ) +from ..limit import limit_websocket class LedgerEventClientManager: websocket: WebSocket - subscriptions: dict[str, List[str]] = {} # [filter, List[subId]] + subscriptions: dict[ + JSONRPCSubscriptionKinds, dict[str, List[str]] + ] = {} # [kind, [filter, List[subId]]] max_subscriptions = 100 def __init__(self, websocket: WebSocket): @@ -115,7 +117,9 @@ async def _handle_request(self, data: JSONRPCRequest) -> JSONRPCResponse: logger.info(f"Received message: {data}") if data.method == JSONRPCMethods.SUBSCRIBE.value: subscribe_params = JSONRPCSubscribeParams.parse_obj(data.params) - self.add_subscription(subscribe_params.filters, subscribe_params.subId) + self.add_subscription( + subscribe_params.kind, subscribe_params.filters, subscribe_params.subId + ) result = JSONRRPCSubscribeResponse( status=JSONRPCStatus.OK, subId=subscribe_params.subId, @@ -146,20 +150,35 @@ async def _send_msg( logger.info(f"Sending message: {data}") await self.websocket.send_text(data.json()) - def add_subscription(self, filters: List[str], subId: str) -> None: - if len(self.subscriptions) >= self.max_subscriptions: + # def handle_subscription_init(self, kind: JSONRPCSubscriptionKinds, filters: List[str], subId: str) -> None: + # async def handle_mint_quote(quote_id: str) -> MintQuote: + # await + # pass + + def add_subscription( + self, kind: JSONRPCSubscriptionKinds, filters: List[str], subId: str + ) -> None: + if kind not in self.subscriptions: + self.subscriptions[kind] = {} + + if len(self.subscriptions[kind]) >= self.max_subscriptions: raise ValueError("Max subscriptions reached") + for filter in filters: if filter not in self.subscriptions: - self.subscriptions[filter] = [] + self.subscriptions[kind][filter] = [] logger.debug(f"Adding subscription {subId} for filter {filter}") - self.subscriptions[filter].append(subId) + self.subscriptions[kind][filter].append(subId) + # self.handle_subscription_init(kind, filters, subId) def remove_subscription(self, subId: str) -> None: - for filter, subs in self.subscriptions.items(): - for sub in subs: - if sub == subId: - logger.debug(f"Removing subscription {subId} for filter {filter}") - self.subscriptions[filter].remove(sub) - return + for kind, sub_filters in self.subscriptions.items(): + for filter, subs in sub_filters.items(): + for sub in subs: + if sub == subId: + logger.debug( + f"Removing subscription {subId} for filter {filter}" + ) + self.subscriptions[kind][filter].remove(sub) + return raise ValueError(f"Subscription not found: {subId}") diff --git a/cashu/mint/events/events.py b/cashu/mint/events/events.py index 01c12d15..3e7db3fd 100644 --- a/cashu/mint/events/events.py +++ b/cashu/mint/events/events.py @@ -3,6 +3,7 @@ from loguru import logger from pydantic import BaseModel +from ...core.json_rpc.base import JSONRPCSubscriptionKinds from .client_manager import LedgerEventClientManager @@ -16,6 +17,11 @@ class LedgerEvent(ABC, BaseModel): def identifier(self) -> str: pass + @property + @abstractmethod + def kind(self) -> JSONRPCSubscriptionKinds: + pass + class LedgerEventManager: """LedgerEventManager is a subscription service from the mint @@ -47,7 +53,8 @@ async def submit(self, event: LedgerEvent) -> None: raise ValueError(f"Unsupported event object type {type(event)}") for client in self.clients: - for sub in client.subscriptions.get(event.identifier, []): + kind_sub = client.subscriptions.get(event.kind, {}) + for sub in kind_sub.get(event.identifier, []): logger.trace( f"Submitting event to sub {sub}: {self.serialize_event(event)}" ) diff --git a/cashu/wallet/subscriptions.py b/cashu/wallet/subscriptions.py index 970477f0..af70f373 100644 --- a/cashu/wallet/subscriptions.py +++ b/cashu/wallet/subscriptions.py @@ -7,11 +7,13 @@ from ..core.crypto.keys import random_hash from ..core.json_rpc.base import ( + JSONRPCMethods, JSONRPCNotficationParams, JSONRPCNotification, JSONRPCRequest, JSONRPCResponse, JSONRPCSubscribeParams, + JSONRPCSubscriptionKinds, JSONRPCUnsubscribeParams, ) @@ -61,7 +63,7 @@ def close(self): # unsubscribe from all subscriptions for subId in self.callback_map.keys(): req = JSONRPCRequest( - method="unsubscribe", + method=JSONRPCMethods.UNSUBSCRIBE.value, params=JSONRPCUnsubscribeParams(subId=subId).dict(), id=self.id_counter, ) @@ -75,12 +77,16 @@ def wait_until_connected(self): while not self.websocket.sock or not self.websocket.sock.connected: time.sleep(0.025) - def subscribe(self, filters: List[str], callback: Callable): + def subscribe( + self, kind: JSONRPCSubscriptionKinds, filters: List[str], callback: Callable + ): self.wait_until_connected() subId = random_hash() req = JSONRPCRequest( - method="subscribe", - params=JSONRPCSubscribeParams(filters=filters, subId=subId).dict(), + method=JSONRPCMethods.SUBSCRIBE.value, + params=JSONRPCSubscribeParams( + kind=kind, filters=filters, subId=subId + ).dict(), id=self.id_counter, ) self.websocket.send(req.json()) diff --git a/cashu/wallet/wallet.py b/cashu/wallet/wallet.py index c253258c..7fc2780d 100644 --- a/cashu/wallet/wallet.py +++ b/cashu/wallet/wallet.py @@ -14,6 +14,8 @@ from httpx import Response from loguru import logger +from cashu.core.json_rpc.base import JSONRPCSubscriptionKinds + from ..core.base import ( BlindedMessage, BlindedSignature, @@ -826,7 +828,11 @@ async def request_mint_subscription( threading.Thread( target=subscriptions.connect, name="SubscriptionManager", daemon=True ).start() - subscriptions.subscribe(filters=[mint_qoute.quote], callback=callback) + subscriptions.subscribe( + kind=JSONRPCSubscriptionKinds.BOLT11_MINT_QUOTE, + filters=[mint_qoute.quote], + callback=callback, + ) # return the invoice decoded_invoice = bolt11.decode(mint_qoute.request) invoice = Invoice( diff --git a/tests/test_wallet_subscriptions.py b/tests/test_wallet_subscriptions.py index 757a371c..acaecbdc 100644 --- a/tests/test_wallet_subscriptions.py +++ b/tests/test_wallet_subscriptions.py @@ -38,7 +38,8 @@ def callback(msg: JSONRPCNotficationParams): invoice, sub = await wallet.request_mint_subscription(128, callback=callback) pay_if_regtest(invoice.bolt11) - await asyncio.sleep(settings.fakewallet_delay_incoming_payment or 3) + wait = settings.fakewallet_delay_incoming_payment or 2 + await asyncio.sleep(2 * wait) assert triggered assert len(msg_stack) == 2 From 6c14452badf674b9ff00659fd02a2066a2392a7a Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Wed, 17 Apr 2024 18:04:10 +0200 Subject: [PATCH 24/45] doesnt start --- cashu/mint/events/client_manager.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/cashu/mint/events/client_manager.py b/cashu/mint/events/client_manager.py index 1c89be92..6e96706f 100644 --- a/cashu/mint/events/client_manager.py +++ b/cashu/mint/events/client_manager.py @@ -1,5 +1,5 @@ import json -from typing import List, Union +from typing import List, Optional, Union from fastapi import WebSocket from loguru import logger @@ -19,19 +19,24 @@ JSONRPCUnsubscribeParams, JSONRRPCSubscribeResponse, ) +from ...mint.ledger import Ledger from ..limit import limit_websocket +from .protocols import SupportsLedger +from .subscription_init import LedgerEventSubscriptionInint -class LedgerEventClientManager: +class LedgerEventClientManager(LedgerEventSubscriptionInint, SupportsLedger): websocket: WebSocket subscriptions: dict[ JSONRPCSubscriptionKinds, dict[str, List[str]] ] = {} # [kind, [filter, List[subId]]] max_subscriptions = 100 + ledger: Optional[Ledger] = None - def __init__(self, websocket: WebSocket): + def __init__(self, websocket: WebSocket, ledger: Optional[Ledger] = None): self.websocket = websocket self.subscriptions = {} + self.ledger = ledger async def start(self): await self.websocket.accept() @@ -150,11 +155,6 @@ async def _send_msg( logger.info(f"Sending message: {data}") await self.websocket.send_text(data.json()) - # def handle_subscription_init(self, kind: JSONRPCSubscriptionKinds, filters: List[str], subId: str) -> None: - # async def handle_mint_quote(quote_id: str) -> MintQuote: - # await - # pass - def add_subscription( self, kind: JSONRPCSubscriptionKinds, filters: List[str], subId: str ) -> None: @@ -169,7 +169,7 @@ def add_subscription( self.subscriptions[kind][filter] = [] logger.debug(f"Adding subscription {subId} for filter {filter}") self.subscriptions[kind][filter].append(subId) - # self.handle_subscription_init(kind, filters, subId) + self.handle_subscription_init(kind, filters, subId) def remove_subscription(self, subId: str) -> None: for kind, sub_filters in self.subscriptions.items(): From 23ec5cf1eaa4ba7f8a0d87065fe2739d91a7d677 Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Thu, 18 Apr 2024 19:28:49 +0200 Subject: [PATCH 25/45] remove circular import --- cashu/mint/events/client_manager.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/cashu/mint/events/client_manager.py b/cashu/mint/events/client_manager.py index 6e96706f..80e0132c 100644 --- a/cashu/mint/events/client_manager.py +++ b/cashu/mint/events/client_manager.py @@ -1,5 +1,5 @@ import json -from typing import List, Optional, Union +from typing import List, Union from fastapi import WebSocket from loguru import logger @@ -19,24 +19,19 @@ JSONRPCUnsubscribeParams, JSONRRPCSubscribeResponse, ) -from ...mint.ledger import Ledger from ..limit import limit_websocket -from .protocols import SupportsLedger -from .subscription_init import LedgerEventSubscriptionInint -class LedgerEventClientManager(LedgerEventSubscriptionInint, SupportsLedger): +class LedgerEventClientManager: websocket: WebSocket subscriptions: dict[ JSONRPCSubscriptionKinds, dict[str, List[str]] ] = {} # [kind, [filter, List[subId]]] max_subscriptions = 100 - ledger: Optional[Ledger] = None - def __init__(self, websocket: WebSocket, ledger: Optional[Ledger] = None): + def __init__(self, websocket: WebSocket): self.websocket = websocket self.subscriptions = {} - self.ledger = ledger async def start(self): await self.websocket.accept() @@ -169,7 +164,7 @@ def add_subscription( self.subscriptions[kind][filter] = [] logger.debug(f"Adding subscription {subId} for filter {filter}") self.subscriptions[kind][filter].append(subId) - self.handle_subscription_init(kind, filters, subId) + # self.handle_subscription_init(kind, filters, subId) def remove_subscription(self, subId: str) -> None: for kind, sub_filters in self.subscriptions.items(): From ed48fe61e6b29af2067fbbf690a904e7f425d881 Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Sun, 16 Jun 2024 21:18:56 +0200 Subject: [PATCH 26/45] update --- cashu/core/base.py | 20 ++++++++++++++++++-- cashu/wallet/cli/cli.py | 2 +- tests/test_wallet_subscriptions.py | 4 ++-- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/cashu/core/base.py b/cashu/core/base.py index a10e4ff3..47cd63d6 100644 --- a/cashu/core/base.py +++ b/cashu/core/base.py @@ -7,7 +7,7 @@ from typing import Dict, List, Optional, Union from loguru import logger -from pydantic import BaseModel +from pydantic import BaseModel, root_validator from cashu.core.json_rpc.base import JSONRPCSubscriptionKinds @@ -57,11 +57,27 @@ def __str__(self): return self.name -class ProofState(BaseModel): +class ProofState(LedgerEvent): Y: str state: SpentState witness: Optional[str] = None + @root_validator() + def check_witness(cls, values): + state, witness = values.get("state"), values.get("witness") + if witness is not None and state != SpentState.spent: + raise ValueError('Witness can only be set if the spent state is "SPENT"') + return values + + @property + def identifier(self) -> str: + """Implementation of the abstract method from LedgerEventManager""" + return self.Y + + @property + def kind(self) -> JSONRPCSubscriptionKinds: + return JSONRPCSubscriptionKinds.PROOF_STATE + class HTLCWitness(BaseModel): preimage: Optional[str] = None diff --git a/cashu/wallet/cli/cli.py b/cashu/wallet/cli/cli.py index b954f919..a80f0760 100644 --- a/cashu/wallet/cli/cli.py +++ b/cashu/wallet/cli/cli.py @@ -314,7 +314,7 @@ def mint_invoice_callback(msg: JSONRPCNotficationParams): # user requests an invoice if amount and not id: - mint_supports_websockets = wallet.mint_info.supports(WEBSOCKETS_NUT) + mint_supports_websockets = wallet.mint_info.supports_nut(WEBSOCKETS_NUT) if mint_supports_websockets: invoice, subscription = await wallet.request_mint_subscription( amount, callback=mint_invoice_callback diff --git a/tests/test_wallet_subscriptions.py b/tests/test_wallet_subscriptions.py index acaecbdc..b9efd599 100644 --- a/tests/test_wallet_subscriptions.py +++ b/tests/test_wallet_subscriptions.py @@ -26,7 +26,7 @@ async def wallet(mint): @pytest.mark.asyncio async def test_wallet_subscription(wallet: Wallet): - assert wallet.mint_info.supports(WEBSOCKETS_NUT) + assert wallet.mint_info.supports_nut(WEBSOCKETS_NUT) triggered = False msg_stack: list[JSONRPCNotficationParams] = [] @@ -39,7 +39,7 @@ def callback(msg: JSONRPCNotficationParams): invoice, sub = await wallet.request_mint_subscription(128, callback=callback) pay_if_regtest(invoice.bolt11) wait = settings.fakewallet_delay_incoming_payment or 2 - await asyncio.sleep(2 * wait) + await asyncio.sleep(wait + 2) assert triggered assert len(msg_stack) == 2 From 30e55c0f8690509816402b2171222b0db9376cbf Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Sun, 16 Jun 2024 21:22:37 +0200 Subject: [PATCH 27/45] fix mypy --- cashu/mint/events/client_manager.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/cashu/mint/events/client_manager.py b/cashu/mint/events/client_manager.py index 80e0132c..037dd250 100644 --- a/cashu/mint/events/client_manager.py +++ b/cashu/mint/events/client_manager.py @@ -43,14 +43,14 @@ async def start(self): limit_websocket(self.websocket) except Exception as e: logger.error(f"Error: {e}") - resp = JSONRPCErrorResponse( + err = JSONRPCErrorResponse( error=JSONRPCError( code=JSONRPCErrorCode.SERVER_ERROR, message=f"Error: {e}", ), id=0, ) - await self._send_msg(resp) + await self._send_msg(err) continue # Parse the JSON data @@ -58,28 +58,28 @@ async def start(self): data = json.loads(json_data) except json.JSONDecodeError as e: logger.error(f"Error decoding JSON: {e}") - resp = JSONRPCErrorResponse( + err = JSONRPCErrorResponse( error=JSONRPCError( code=JSONRPCErrorCode.PARSE_ERROR, message=f"Error: {e}", ), id=0, ) - await self._send_msg(resp) + await self._send_msg(err) continue # Parse the JSONRPCRequest try: req = JSONRPCRequest.parse_obj(data) except Exception as e: - resp = JSONRPCErrorResponse( + err = JSONRPCErrorResponse( error=JSONRPCError( code=JSONRPCErrorCode.INVALID_REQUEST, message=f"Error: {e}", ), id=0, ) - await self._send_msg(resp) + await self._send_msg(err) logger.warning(f"Error handling websocket message: {e}") continue @@ -87,14 +87,14 @@ async def start(self): try: JSONRPCMethods(req.method) except ValueError: - resp = JSONRPCErrorResponse( + err = JSONRPCErrorResponse( error=JSONRPCError( code=JSONRPCErrorCode.METHOD_NOT_FOUND, message=f"Method not found: {req.method}", ), id=req.id, ) - await self._send_msg(resp) + await self._send_msg(err) continue # Handle the request @@ -102,13 +102,15 @@ async def start(self): logger.debug(f"Request: {req}") resp = await self._handle_request(req) except Exception as e: - resp = JSONRPCErrorResponse( + err = JSONRPCErrorResponse( error=JSONRPCError( code=JSONRPCErrorCode.INTERNAL_ERROR, message=f"Error: {e}", ), id=req.id, ) + await self._send_msg(err) + raise e # Send the response await self._send_msg(resp) From 2ee20340994b22e7744cf9ad95538325797b0cc2 Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Sun, 16 Jun 2024 22:24:49 +0200 Subject: [PATCH 28/45] move test file in test because it fails if it runs later... dunno why --- cashu/mint/events/client_manager.py | 7 +++---- cashu/wallet/cli/cli.py | 2 +- cashu/wallet/wallet.py | 4 ++-- ...llet_subscriptions.py => test_a_wallet_subscription.py} | 2 +- tests/test_mint_fees.py | 2 +- 5 files changed, 8 insertions(+), 9 deletions(-) rename tests/{test_wallet_subscriptions.py => test_a_wallet_subscription.py} (94%) diff --git a/cashu/mint/events/client_manager.py b/cashu/mint/events/client_manager.py index 037dd250..0d702477 100644 --- a/cashu/mint/events/client_manager.py +++ b/cashu/mint/events/client_manager.py @@ -101,6 +101,8 @@ async def start(self): try: logger.debug(f"Request: {req}") resp = await self._handle_request(req) + # Send the response + await self._send_msg(resp) except Exception as e: err = JSONRPCErrorResponse( error=JSONRPCError( @@ -110,10 +112,7 @@ async def start(self): id=req.id, ) await self._send_msg(err) - raise e - - # Send the response - await self._send_msg(resp) + continue async def _handle_request(self, data: JSONRPCRequest) -> JSONRPCResponse: logger.info(f"Received message: {data}") diff --git a/cashu/wallet/cli/cli.py b/cashu/wallet/cli/cli.py index a80f0760..22da01d1 100644 --- a/cashu/wallet/cli/cli.py +++ b/cashu/wallet/cli/cli.py @@ -316,7 +316,7 @@ def mint_invoice_callback(msg: JSONRPCNotficationParams): if amount and not id: mint_supports_websockets = wallet.mint_info.supports_nut(WEBSOCKETS_NUT) if mint_supports_websockets: - invoice, subscription = await wallet.request_mint_subscription( + invoice, subscription = await wallet.request_mint_with_callback( amount, callback=mint_invoice_callback ) invoice_nonlocal, subscription_nonlocal = invoice, subscription diff --git a/cashu/wallet/wallet.py b/cashu/wallet/wallet.py index 9d0b8f5b..73971bea 100644 --- a/cashu/wallet/wallet.py +++ b/cashu/wallet/wallet.py @@ -316,14 +316,14 @@ async def _check_used_secrets(self, secrets): raise Exception(f"secret already used: {s}") logger.trace("Secret check complete.") - async def request_mint_subscription( + async def request_mint_with_callback( self, amount: int, callback: Callable ) -> Tuple[Invoice, SubscriptionManager]: """Request a Lightning invoice for minting tokens. Args: amount (int): Amount for Lightning invoice in satoshis - callback (Optional[Callable], optional): Callback function to be called when the invoice is paid. Defaults to None. + callback (Callable): Callback function to be called when the invoice is paid. Returns: Invoice: Lightning invoice diff --git a/tests/test_wallet_subscriptions.py b/tests/test_a_wallet_subscription.py similarity index 94% rename from tests/test_wallet_subscriptions.py rename to tests/test_a_wallet_subscription.py index b9efd599..7de3d1fa 100644 --- a/tests/test_wallet_subscriptions.py +++ b/tests/test_a_wallet_subscription.py @@ -36,7 +36,7 @@ def callback(msg: JSONRPCNotficationParams): msg_stack.append(msg) asyncio.run(wallet.mint(int(invoice.amount), id=invoice.id)) - invoice, sub = await wallet.request_mint_subscription(128, callback=callback) + invoice, sub = await wallet.request_mint_with_callback(128, callback=callback) pay_if_regtest(invoice.bolt11) wait = settings.fakewallet_delay_incoming_payment or 2 await asyncio.sleep(wait + 2) diff --git a/tests/test_mint_fees.py b/tests/test_mint_fees.py index 106d1fbd..673989bf 100644 --- a/tests/test_mint_fees.py +++ b/tests/test_mint_fees.py @@ -95,7 +95,7 @@ async def test_get_fees_for_proofs(wallet1: Wallet, ledger: Ledger): @pytest.mark.asyncio -@pytest.mark.skipif_with_fees(is_regtest, reason="only works with FakeWallet") +@pytest.mark.skipif(is_regtest, reason="only works with FakeWallet") async def test_wallet_fee(wallet1: Wallet, ledger: Ledger): # THIS TEST IS A FAKE, WE SET THE WALLET FEES MANUALLY IN set_ledger_keyset_fees # It would be better to test if the wallet can get the fees from the mint itself From d218d7c0d121913cde06fa684e4371f281a99bea Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Mon, 17 Jun 2024 14:54:14 +0200 Subject: [PATCH 29/45] adjust websocket NUT-06 settings --- cashu/core/nuts.py | 2 +- cashu/lightning/base.py | 1 + cashu/lightning/corelightningrest.py | 1 + cashu/lightning/fake.py | 2 ++ cashu/lightning/lndrest.py | 1 + cashu/mint/events/client_manager.py | 5 ++--- cashu/mint/features.py | 27 ++++++++++++++++++++++++++- cashu/wallet/cli/cli.py | 7 ++++--- cashu/wallet/mint_info.py | 20 +++++++++++++++++--- tests/test_a_wallet_subscription.py | 7 ++++++- 10 files changed, 61 insertions(+), 12 deletions(-) diff --git a/cashu/core/nuts.py b/cashu/core/nuts.py index bf41745c..6c0b5287 100644 --- a/cashu/core/nuts.py +++ b/cashu/core/nuts.py @@ -9,5 +9,5 @@ P2PK_NUT = 11 DLEQ_NUT = 12 DETERMINSTIC_SECRETS_NUT = 13 -WEBSOCKETS_NUT = 14 MPP_NUT = 15 +WEBSOCKETS_NUT = 17 diff --git a/cashu/lightning/base.py b/cashu/lightning/base.py index 8cb8574d..06fbe975 100644 --- a/cashu/lightning/base.py +++ b/cashu/lightning/base.py @@ -68,6 +68,7 @@ def __str__(self) -> str: class LightningBackend(ABC): supports_mpp: bool = False + supports_incoming_payment_stream: bool = False supported_units: set[Unit] unit: Unit diff --git a/cashu/lightning/corelightningrest.py b/cashu/lightning/corelightningrest.py index ccc06772..ce62a51f 100644 --- a/cashu/lightning/corelightningrest.py +++ b/cashu/lightning/corelightningrest.py @@ -29,6 +29,7 @@ class CoreLightningRestWallet(LightningBackend): supported_units = set([Unit.sat, Unit.msat]) unit = Unit.sat + supports_incoming_payment_stream: bool = True def __init__(self, unit: Unit = Unit.sat, **kwargs): self.assert_unit_supported(unit) diff --git a/cashu/lightning/fake.py b/cashu/lightning/fake.py index 7ce43622..b9eccbd1 100644 --- a/cashu/lightning/fake.py +++ b/cashu/lightning/fake.py @@ -48,6 +48,8 @@ class FakeWallet(LightningBackend): supported_units = set([Unit.sat, Unit.msat, Unit.usd]) unit = Unit.sat + supports_incoming_payment_stream: bool = True + def __init__(self, unit: Unit = Unit.sat, **kwargs): self.assert_unit_supported(unit) self.unit = unit diff --git a/cashu/lightning/lndrest.py b/cashu/lightning/lndrest.py index 477e84f4..e534f353 100644 --- a/cashu/lightning/lndrest.py +++ b/cashu/lightning/lndrest.py @@ -31,6 +31,7 @@ class LndRestWallet(LightningBackend): """https://api.lightning.community/rest/index.html#lnd-rest-api-reference""" supports_mpp = settings.mint_lnd_enable_mpp + supports_incoming_payment_stream = True supported_units = set([Unit.sat, Unit.msat]) unit = Unit.sat diff --git a/cashu/mint/events/client_manager.py b/cashu/mint/events/client_manager.py index 0d702477..17ff7dea 100644 --- a/cashu/mint/events/client_manager.py +++ b/cashu/mint/events/client_manager.py @@ -115,7 +115,7 @@ async def start(self): continue async def _handle_request(self, data: JSONRPCRequest) -> JSONRPCResponse: - logger.info(f"Received message: {data}") + logger.debug(f"Received websocket message: {data}") if data.method == JSONRPCMethods.SUBSCRIBE.value: subscribe_params = JSONRPCSubscribeParams.parse_obj(data.params) self.add_subscription( @@ -138,7 +138,6 @@ async def _handle_request(self, data: JSONRPCRequest) -> JSONRPCResponse: raise ValueError(f"Invalid method: {data.method}") async def _send_obj(self, data: dict, subId: str): - logger.info(f"Sending object: {data}") resp = JSONRPCNotification( method=JSONRPCMethods.SUBSCRIBE.value, params=JSONRPCNotficationParams(subId=subId, payload=data).dict(), @@ -148,7 +147,7 @@ async def _send_obj(self, data: dict, subId: str): async def _send_msg( self, data: Union[JSONRPCResponse, JSONRPCNotification, JSONRPCErrorResponse] ): - logger.info(f"Sending message: {data}") + logger.debug(f"Sending websocket message: {data}") await self.websocket.send_text(data.json()) def add_subscription( diff --git a/cashu/mint/features.py b/cashu/mint/features.py index bcc42233..439cceac 100644 --- a/cashu/mint/features.py +++ b/cashu/mint/features.py @@ -1,5 +1,6 @@ from typing import Any, Dict, List, Union +from cashu.core.base import Method from cashu.mint.protocols import SupportsBackends from ..core.models import ( @@ -56,7 +57,6 @@ def mint_features(self) -> Dict[int, Union[List[Any], Dict[str, Any]]]: SCRIPT_NUT: supported_dict, P2PK_NUT: supported_dict, DLEQ_NUT: supported_dict, - WEBSOCKETS_NUT: supported_dict, } # signal which method-unit pairs support MPP @@ -75,4 +75,29 @@ def mint_features(self) -> Dict[int, Union[List[Any], Dict[str, Any]]]: if mpp_features: mint_features[MPP_NUT] = mpp_features + # specify which websocket features are supported + # these two are supported by default + websocket_features: List[Dict[str, Union[str, List[str]]]] = [] + # we check the backend to see if "bolt11_mint_quote" is supported as well + for method, unit_dict in self.backends.items(): + if method == Method["bolt11"]: + for unit in unit_dict.keys(): + websocket_features.append( + { + "method": method.name, + "unit": unit.name, + "commands": ["bolt11_melt_quote", "proof_state"], + } + ) + if unit_dict[unit].supports_incoming_payment_stream: + supported_features: List[str] = list( + websocket_features[-1]["commands"] + ) + websocket_features[-1]["commands"] = supported_features + [ + "bolt11_mint_quote" + ] + + if websocket_features: + mint_features[WEBSOCKETS_NUT] = websocket_features + return mint_features diff --git a/cashu/wallet/cli/cli.py b/cashu/wallet/cli/cli.py index 22da01d1..5859e5b8 100644 --- a/cashu/wallet/cli/cli.py +++ b/cashu/wallet/cli/cli.py @@ -15,11 +15,10 @@ from click import Context from loguru import logger -from ...core.base import Invoice, MintQuote, TokenV3, Unit +from ...core.base import Invoice, Method, MintQuote, TokenV3, Unit from ...core.helpers import sum_proofs from ...core.json_rpc.base import JSONRPCNotficationParams from ...core.logging import configure_logger -from ...core.nuts import WEBSOCKETS_NUT from ...core.settings import settings from ...nostr.client.client import NostrClient from ...tor.tor import TorProxy @@ -314,7 +313,9 @@ def mint_invoice_callback(msg: JSONRPCNotficationParams): # user requests an invoice if amount and not id: - mint_supports_websockets = wallet.mint_info.supports_nut(WEBSOCKETS_NUT) + mint_supports_websockets = wallet.mint_info.supports_websocket_mint_quote( + Method["bolt11"], wallet.unit + ) if mint_supports_websockets: invoice, subscription = await wallet.request_mint_with_callback( amount, callback=mint_invoice_callback diff --git a/cashu/wallet/mint_info.py b/cashu/wallet/mint_info.py index e9092dd6..84035beb 100644 --- a/cashu/wallet/mint_info.py +++ b/cashu/wallet/mint_info.py @@ -2,7 +2,9 @@ from pydantic import BaseModel -from ..core.base import Unit +from cashu.core.nuts import MPP_NUT, WEBSOCKETS_NUT + +from ..core.base import Method, Unit from ..core.models import Nut15MppSupport @@ -27,8 +29,8 @@ def supports_nut(self, nut: int) -> bool: def supports_mpp(self, method: str, unit: Unit) -> bool: if not self.nuts: return False - nut_15 = self.nuts.get(15) - if not nut_15 or not self.supports_nut(15): + nut_15 = self.nuts.get(MPP_NUT) + if not nut_15 or not self.supports_nut(MPP_NUT): return False for entry in nut_15: @@ -37,3 +39,15 @@ def supports_mpp(self, method: str, unit: Unit) -> bool: return True return False + + def supports_websocket_mint_quote(self, method: Method, unit: Unit) -> bool: + if not self.nuts or not self.supports_nut(WEBSOCKETS_NUT): + return False + websocket_settings = self.nuts[WEBSOCKETS_NUT] + if not websocket_settings: + return False + for entry in websocket_settings: + if entry["method"] == method.name and entry["unit"] == unit.name: + if "bolt11_mint_quote" in entry["commands"]: + return True + return False diff --git a/tests/test_a_wallet_subscription.py b/tests/test_a_wallet_subscription.py index 7de3d1fa..c1ca9414 100644 --- a/tests/test_a_wallet_subscription.py +++ b/tests/test_a_wallet_subscription.py @@ -26,7 +26,12 @@ async def wallet(mint): @pytest.mark.asyncio async def test_wallet_subscription(wallet: Wallet): - assert wallet.mint_info.supports_nut(WEBSOCKETS_NUT) + if not wallet.mint_info.supports_nut(WEBSOCKETS_NUT): + pytest.skip("No websocket support") + + if not wallet.mint_info.supports_websocket_bolt11_mint_quote(): + pytest.skip("No websocket support for bolt11_mint_quote") + triggered = False msg_stack: list[JSONRPCNotficationParams] = [] From 7cece91c8bc7a83df8ab12351c9808eacf4927c2 Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Mon, 17 Jun 2024 14:56:07 +0200 Subject: [PATCH 30/45] local import and small fix --- cashu/mint/features.py | 5 ++--- tests/test_a_wallet_subscription.py | 5 ++++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/cashu/mint/features.py b/cashu/mint/features.py index 439cceac..494033e8 100644 --- a/cashu/mint/features.py +++ b/cashu/mint/features.py @@ -1,8 +1,6 @@ from typing import Any, Dict, List, Union -from cashu.core.base import Method -from cashu.mint.protocols import SupportsBackends - +from ..core.base import Method from ..core.models import ( MintMeltMethodSetting, ) @@ -19,6 +17,7 @@ WEBSOCKETS_NUT, ) from ..core.settings import settings +from ..mint.protocols import SupportsBackends class LedgerFeatures(SupportsBackends): diff --git a/tests/test_a_wallet_subscription.py b/tests/test_a_wallet_subscription.py index c1ca9414..f6f81a0e 100644 --- a/tests/test_a_wallet_subscription.py +++ b/tests/test_a_wallet_subscription.py @@ -3,6 +3,7 @@ import pytest import pytest_asyncio +from cashu.core.base import Method from cashu.core.json_rpc.base import JSONRPCNotficationParams from cashu.core.nuts import WEBSOCKETS_NUT from cashu.core.settings import settings @@ -29,7 +30,9 @@ async def test_wallet_subscription(wallet: Wallet): if not wallet.mint_info.supports_nut(WEBSOCKETS_NUT): pytest.skip("No websocket support") - if not wallet.mint_info.supports_websocket_bolt11_mint_quote(): + if not wallet.mint_info.supports_websocket_mint_quote( + Method["bolt11"], wallet.unit + ): pytest.skip("No websocket support for bolt11_mint_quote") triggered = False From d3ebf7975c5c8b80d9f43f321afb6cfcc87dde35 Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Mon, 17 Jun 2024 16:46:53 +0200 Subject: [PATCH 31/45] disable websockets in CLI if "no_check" is selected --- cashu/mint/verification.py | 7 +++++-- cashu/wallet/cli/cli.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/cashu/mint/verification.py b/cashu/mint/verification.py index 68bbbc42..62275165 100644 --- a/cashu/mint/verification.py +++ b/cashu/mint/verification.py @@ -128,11 +128,14 @@ async def _verify_outputs( if not self._verify_no_duplicate_outputs(outputs): raise TransactionError("duplicate outputs.") # verify that outputs have not been signed previously - if any(await self._check_outputs_issued_before(outputs)): + signed_before = await self._check_outputs_issued_before(outputs) + if any(signed_before): raise TransactionError("outputs have already been signed before.") logger.trace(f"Verified {len(outputs)} outputs.") - async def _check_outputs_issued_before(self, outputs: List[BlindedMessage]): + async def _check_outputs_issued_before( + self, outputs: List[BlindedMessage] + ) -> List[bool]: """Checks whether the provided outputs have previously been signed by the mint (which would lead to a duplication error later when trying to store these outputs again). diff --git a/cashu/wallet/cli/cli.py b/cashu/wallet/cli/cli.py index 5859e5b8..0f3a2259 100644 --- a/cashu/wallet/cli/cli.py +++ b/cashu/wallet/cli/cli.py @@ -316,7 +316,7 @@ def mint_invoice_callback(msg: JSONRPCNotficationParams): mint_supports_websockets = wallet.mint_info.supports_websocket_mint_quote( Method["bolt11"], wallet.unit ) - if mint_supports_websockets: + if mint_supports_websockets and not no_check: invoice, subscription = await wallet.request_mint_with_callback( amount, callback=mint_invoice_callback ) From 03c7e36fc1f7958eb6cbd01f9f23aa4bf93be8b0 Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Mon, 17 Jun 2024 16:55:34 +0200 Subject: [PATCH 32/45] move subscription test to where it was --- ...{test_a_wallet_subscription.py => test_wallet_subscription.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{test_a_wallet_subscription.py => test_wallet_subscription.py} (100%) diff --git a/tests/test_a_wallet_subscription.py b/tests/test_wallet_subscription.py similarity index 100% rename from tests/test_a_wallet_subscription.py rename to tests/test_wallet_subscription.py From 5c03c3db4b3a5b17151ba119fb2fcafe48c5db36 Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Tue, 18 Jun 2024 14:07:48 +0200 Subject: [PATCH 33/45] check proof state with callback, add tests --- cashu/mint/events/client_manager.py | 3 +- cashu/wallet/wallet.py | 16 ++++++++ tests/test_wallet_subscription.py | 62 +++++++++++++++++++++++++---- 3 files changed, 72 insertions(+), 9 deletions(-) diff --git a/cashu/mint/events/client_manager.py b/cashu/mint/events/client_manager.py index 17ff7dea..2da07faa 100644 --- a/cashu/mint/events/client_manager.py +++ b/cashu/mint/events/client_manager.py @@ -27,7 +27,7 @@ class LedgerEventClientManager: subscriptions: dict[ JSONRPCSubscriptionKinds, dict[str, List[str]] ] = {} # [kind, [filter, List[subId]]] - max_subscriptions = 100 + max_subscriptions = 1000 def __init__(self, websocket: WebSocket): self.websocket = websocket @@ -164,7 +164,6 @@ def add_subscription( self.subscriptions[kind][filter] = [] logger.debug(f"Adding subscription {subId} for filter {filter}") self.subscriptions[kind][filter].append(subId) - # self.handle_subscription_init(kind, filters, subId) def remove_subscription(self, subId: str) -> None: for kind, sub_filters in self.subscriptions.items(): diff --git a/cashu/wallet/wallet.py b/cashu/wallet/wallet.py index 73971bea..2426f024 100644 --- a/cashu/wallet/wallet.py +++ b/cashu/wallet/wallet.py @@ -724,6 +724,20 @@ async def melt( async def check_proof_state(self, proofs) -> PostCheckStateResponse: return await super().check_proof_state(proofs) + async def check_proof_state_with_callback( + self, proofs: List[Proof], callback: Callable + ) -> Tuple[PostCheckStateResponse, SubscriptionManager]: + subscriptions = SubscriptionManager(self.url) + threading.Thread( + target=subscriptions.connect, name="SubscriptionManager", daemon=True + ).start() + subscriptions.subscribe( + kind=JSONRPCSubscriptionKinds.PROOF_STATE, + filters=[proof.Y for proof in proofs], + callback=callback, + ) + return await self.check_proof_state(proofs), subscriptions + # ---------- TOKEN MECHANICS ---------- # ---------- DLEQ PROOFS ---------- @@ -1110,6 +1124,8 @@ async def balance_per_minturl( balances_return[key]["unit"] = unit.name return dict(sorted(balances_return.items(), key=lambda item: item[0])) # type: ignore + # ---------- RESTORE WALLET ---------- + async def restore_tokens_for_keyset( self, keyset_id: str, to: int = 2, batch: int = 25 ) -> None: diff --git a/tests/test_wallet_subscription.py b/tests/test_wallet_subscription.py index f6f81a0e..a99c57fa 100644 --- a/tests/test_wallet_subscription.py +++ b/tests/test_wallet_subscription.py @@ -3,7 +3,7 @@ import pytest import pytest_asyncio -from cashu.core.base import Method +from cashu.core.base import Method, ProofState from cashu.core.json_rpc.base import JSONRPCNotficationParams from cashu.core.nuts import WEBSOCKETS_NUT from cashu.core.settings import settings @@ -26,7 +26,7 @@ async def wallet(mint): @pytest.mark.asyncio -async def test_wallet_subscription(wallet: Wallet): +async def test_wallet_subscription_mint(wallet: Wallet): if not wallet.mint_info.supports_nut(WEBSOCKETS_NUT): pytest.skip("No websocket support") @@ -45,14 +45,62 @@ def callback(msg: JSONRPCNotficationParams): asyncio.run(wallet.mint(int(invoice.amount), id=invoice.id)) invoice, sub = await wallet.request_mint_with_callback(128, callback=callback) - pay_if_regtest(invoice.bolt11) - wait = settings.fakewallet_delay_incoming_payment or 2 - await asyncio.sleep(wait + 2) - assert triggered - assert len(msg_stack) == 2 + # first we expect the issued=False state to arrive + await asyncio.sleep(2) + assert triggered + assert len(msg_stack) == 1 assert msg_stack[0].payload["paid"] is True assert msg_stack[0].payload["issued"] is False + # this will cause a second message + pay_if_regtest(invoice.bolt11) + wait = settings.fakewallet_delay_incoming_payment or 2 + await asyncio.sleep(wait + 2) + assert len(msg_stack) == 2 assert msg_stack[1].payload["paid"] is True assert msg_stack[1].payload["issued"] is True + + +@pytest.mark.asyncio +async def test_wallet_subscription_swap(wallet: Wallet): + if not wallet.mint_info.supports_nut(WEBSOCKETS_NUT): + pytest.skip("No websocket support") + + invoice = await wallet.request_mint(64) + pay_if_regtest(invoice.bolt11) + await wallet.mint(64, id=invoice.id) + + triggered = False + msg_stack: list[JSONRPCNotficationParams] = [] + + def callback(msg: JSONRPCNotficationParams): + nonlocal triggered, msg_stack + triggered = True + msg_stack.append(msg) + + n_subscriptions = len(wallet.proofs) + state, sub = await wallet.check_proof_state_with_callback( + wallet.proofs, callback=callback + ) + + _ = await wallet.split_to_send(wallet.proofs, 64) + + wait = 1 + await asyncio.sleep(wait) + assert triggered + + # we receive 2 messages for each subscription + assert len(msg_stack) == n_subscriptions * 2 + + # the first one is the PENDING state + pending_stack = msg_stack[:n_subscriptions] + for msg in pending_stack: + proof_state = ProofState.parse_obj(msg.payload) + assert proof_state.state.value == "PENDING" + + # the second one is the SPENT state + spent_stack = msg_stack[n_subscriptions:] + for msg in spent_stack: + proof_state = ProofState.parse_obj(msg.payload) + assert proof_state.state.value == "SPENT" From 0ac7c2c50d3393620f79937c4f5a0de85530c972 Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Tue, 18 Jun 2024 15:01:25 +0200 Subject: [PATCH 34/45] tests: run mint fixture per module instead of per session --- tests/conftest.py | 2 +- tests/test_wallet_subscription.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 32f9e4cd..4954d4c6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -127,7 +127,7 @@ async def start_mint_init(ledger: Ledger) -> Ledger: # # This fixture is used for tests that require API access to the mint -@pytest.fixture(autouse=True, scope="session") +@pytest.fixture(autouse=True, scope="module") def mint(): config = uvicorn.Config( "cashu.mint.app:app", diff --git a/tests/test_wallet_subscription.py b/tests/test_wallet_subscription.py index a99c57fa..21100369 100644 --- a/tests/test_wallet_subscription.py +++ b/tests/test_wallet_subscription.py @@ -46,6 +46,8 @@ def callback(msg: JSONRPCNotficationParams): invoice, sub = await wallet.request_mint_with_callback(128, callback=callback) + # TODO: check for pending and paid states according to: https://github.com/cashubtc/nuts/pull/136 + # first we expect the issued=False state to arrive await asyncio.sleep(2) assert triggered From 4f157a35a07f9d9dba98dfce12029e8b5147bb3b Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Wed, 19 Jun 2024 12:57:50 +0200 Subject: [PATCH 35/45] subscription command name fix --- cashu/core/json_rpc/base.py | 4 ++-- cashu/core/settings.py | 6 ++++++ cashu/mint/events/client_manager.py | 15 +++++++++++++-- cashu/mint/router.py | 5 ++++- cashu/wallet/subscriptions.py | 2 ++ 5 files changed, 27 insertions(+), 5 deletions(-) diff --git a/cashu/core/json_rpc/base.py b/cashu/core/json_rpc/base.py index 2b7c7fe2..b293b5af 100644 --- a/cashu/core/json_rpc/base.py +++ b/cashu/core/json_rpc/base.py @@ -52,8 +52,8 @@ class JSONRPCErrorResponse(BaseModel): class JSONRPCMethods(Enum): - SUBSCRIBE = "sub" - UNSUBSCRIBE = "unsub" + SUBSCRIBE = "subscribe" + UNSUBSCRIBE = "unsubscribe" class JSONRPCSubscriptionKinds(Enum): diff --git a/cashu/core/settings.py b/cashu/core/settings.py index 2412534d..23a8c4c4 100644 --- a/cashu/core/settings.py +++ b/cashu/core/settings.py @@ -120,6 +120,12 @@ class MintLimits(MintSettings): title="Maximum mint balance", description="Maximum mint balance.", ) + mint_websocket_read_timeout: int = Field( + default=10 * 60, + gt=0, + title="Websocket read timeout", + description="Timeout for reading from a websocket.", + ) class FakeWalletSettings(MintSettings): diff --git a/cashu/mint/events/client_manager.py b/cashu/mint/events/client_manager.py index 2da07faa..dbf37f7d 100644 --- a/cashu/mint/events/client_manager.py +++ b/cashu/mint/events/client_manager.py @@ -1,3 +1,4 @@ +import asyncio import json from typing import List, Union @@ -19,6 +20,7 @@ JSONRPCUnsubscribeParams, JSONRRPCSubscribeResponse, ) +from ...core.settings import settings from ..limit import limit_websocket @@ -35,8 +37,13 @@ def __init__(self, websocket: WebSocket): async def start(self): await self.websocket.accept() + while True: - json_data = await self.websocket.receive_text() + message = await asyncio.wait_for( + self.websocket.receive(), + timeout=settings.mint_websocket_read_timeout, + ) + message_text = message.get("text") # Check the rate limit try: @@ -53,9 +60,13 @@ async def start(self): await self._send_msg(err) continue + # Check if message contains text + if not message_text: + continue + # Parse the JSON data try: - data = json.loads(json_data) + data = json.loads(message_text) except json.JSONDecodeError as e: logger.error(f"Error decoding JSON: {e}") err = JSONRPCErrorResponse( diff --git a/cashu/mint/router.py b/cashu/mint/router.py index 0a02b680..13702713 100644 --- a/cashu/mint/router.py +++ b/cashu/mint/router.py @@ -1,3 +1,5 @@ +import asyncio + from fastapi import APIRouter, Request, WebSocket from loguru import logger @@ -196,13 +198,14 @@ async def websocket_endpoint(websocket: WebSocket): client = LedgerEventClientManager(websocket=websocket) success = ledger.events.add_client(client) if not success: - await websocket.close() + await asyncio.wait_for(websocket.close(), timeout=1) return try: await client.start() except Exception as e: logger.debug(f"Exception: {e}") ledger.events.remove_client(client) + await asyncio.wait_for(websocket.close(), timeout=1) @router.post( diff --git a/cashu/wallet/subscriptions.py b/cashu/wallet/subscriptions.py index af70f373..35c0b301 100644 --- a/cashu/wallet/subscriptions.py +++ b/cashu/wallet/subscriptions.py @@ -67,6 +67,7 @@ def close(self): params=JSONRPCUnsubscribeParams(subId=subId).dict(), id=self.id_counter, ) + logger.trace(f"Unsubscribing: {req.json()}") self.websocket.send(req.json()) self.id_counter += 1 @@ -89,6 +90,7 @@ def subscribe( ).dict(), id=self.id_counter, ) + logger.trace(f"Subscribing: {req.json()}") self.websocket.send(req.json()) self.id_counter += 1 self.callback_map[subId] = callback From 97072aef9319705eaac8cfcd71c6cb6a31223079 Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Wed, 19 Jun 2024 13:01:21 +0200 Subject: [PATCH 36/45] test per session again --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 4954d4c6..32f9e4cd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -127,7 +127,7 @@ async def start_mint_init(ledger: Ledger) -> Ledger: # # This fixture is used for tests that require API access to the mint -@pytest.fixture(autouse=True, scope="module") +@pytest.fixture(autouse=True, scope="session") def mint(): config = uvicorn.Config( "cashu.mint.app:app", From 9c3f9f76f3a16243d1f54603b380dbccbdf6cde7 Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Wed, 19 Jun 2024 19:46:40 +0200 Subject: [PATCH 37/45] update test race conditions --- tests/conftest.py | 2 +- tests/test_wallet_cli.py | 16 +++++++++++++++- tests/test_wallet_subscription.py | 15 ++++++++++----- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 32f9e4cd..ffed86b5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -34,7 +34,7 @@ settings.mint_backend_bolt11_sat = settings.mint_backend_bolt11_sat or "FakeWallet" settings.fakewallet_brr = True settings.fakewallet_delay_outgoing_payment = None -settings.fakewallet_delay_incoming_payment = 2 +settings.fakewallet_delay_incoming_payment = 5 settings.fakewallet_stochastic_invoice = False assert ( settings.mint_test_database != settings.mint_database diff --git a/tests/test_wallet_cli.py b/tests/test_wallet_cli.py index 42a9d158..ad40cfa2 100644 --- a/tests/test_wallet_cli.py +++ b/tests/test_wallet_cli.py @@ -109,7 +109,7 @@ def test_balance(cli_prefix): assert result.exit_code == 0 -def test_invoice(mint, cli_prefix): +def test_invoice_no_check(mint, cli_prefix): runner = CliRunner() result = runner.invoke( cli, @@ -132,6 +132,20 @@ def test_invoice(mint, cli_prefix): assert result.exit_code == 0 +def test_invoice(mint, cli_prefix): + runner = CliRunner() + result = runner.invoke( + cli, + [*cli_prefix, "invoice", "1000"], + ) + + assert result.exception is None + + wallet = asyncio.run(init_wallet()) + assert wallet.available_balance >= 1000 + assert result.exit_code == 0 + + def test_invoice_with_split(mint, cli_prefix): runner = CliRunner() result = runner.invoke( diff --git a/tests/test_wallet_subscription.py b/tests/test_wallet_subscription.py index 21100369..c42ce885 100644 --- a/tests/test_wallet_subscription.py +++ b/tests/test_wallet_subscription.py @@ -10,6 +10,7 @@ from cashu.wallet.wallet import Wallet from tests.conftest import SERVER_ENDPOINT from tests.helpers import ( + is_fake, pay_if_regtest, ) @@ -27,6 +28,9 @@ async def wallet(mint): @pytest.mark.asyncio async def test_wallet_subscription_mint(wallet: Wallet): + if is_fake: + settings.fakewallet_delay_outgoing_payment = 2 + if not wallet.mint_info.supports_nut(WEBSOCKETS_NUT): pytest.skip("No websocket support") @@ -45,20 +49,21 @@ def callback(msg: JSONRPCNotficationParams): asyncio.run(wallet.mint(int(invoice.amount), id=invoice.id)) invoice, sub = await wallet.request_mint_with_callback(128, callback=callback) + pay_if_regtest(invoice.bolt11) + wait = settings.fakewallet_delay_incoming_payment or 2 + await asyncio.sleep(wait + 2) # TODO: check for pending and paid states according to: https://github.com/cashubtc/nuts/pull/136 # first we expect the issued=False state to arrive - await asyncio.sleep(2) + assert triggered - assert len(msg_stack) == 1 + # assert len(msg_stack) == 1 assert msg_stack[0].payload["paid"] is True assert msg_stack[0].payload["issued"] is False + # await asyncio.sleep(2) # this will cause a second message - pay_if_regtest(invoice.bolt11) - wait = settings.fakewallet_delay_incoming_payment or 2 - await asyncio.sleep(wait + 2) assert len(msg_stack) == 2 assert msg_stack[1].payload["paid"] is True assert msg_stack[1].payload["issued"] is True From ffe699f4c95c12769daf9f3ca929b92711d324f4 Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Wed, 19 Jun 2024 21:10:44 +0200 Subject: [PATCH 38/45] fix tests --- tests/conftest.py | 2 +- tests/test_wallet_cli.py | 16 +--------------- tests/test_wallet_subscription.py | 4 +--- 3 files changed, 3 insertions(+), 19 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index ffed86b5..ed3dc2f1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -34,7 +34,7 @@ settings.mint_backend_bolt11_sat = settings.mint_backend_bolt11_sat or "FakeWallet" settings.fakewallet_brr = True settings.fakewallet_delay_outgoing_payment = None -settings.fakewallet_delay_incoming_payment = 5 +settings.fakewallet_delay_incoming_payment = 1 settings.fakewallet_stochastic_invoice = False assert ( settings.mint_test_database != settings.mint_database diff --git a/tests/test_wallet_cli.py b/tests/test_wallet_cli.py index ad40cfa2..26102da5 100644 --- a/tests/test_wallet_cli.py +++ b/tests/test_wallet_cli.py @@ -109,7 +109,7 @@ def test_balance(cli_prefix): assert result.exit_code == 0 -def test_invoice_no_check(mint, cli_prefix): +def test_invoice_return_immediately(mint, cli_prefix): runner = CliRunner() result = runner.invoke( cli, @@ -132,20 +132,6 @@ def test_invoice_no_check(mint, cli_prefix): assert result.exit_code == 0 -def test_invoice(mint, cli_prefix): - runner = CliRunner() - result = runner.invoke( - cli, - [*cli_prefix, "invoice", "1000"], - ) - - assert result.exception is None - - wallet = asyncio.run(init_wallet()) - assert wallet.available_balance >= 1000 - assert result.exit_code == 0 - - def test_invoice_with_split(mint, cli_prefix): runner = CliRunner() result = runner.invoke( diff --git a/tests/test_wallet_subscription.py b/tests/test_wallet_subscription.py index c42ce885..60c512bb 100644 --- a/tests/test_wallet_subscription.py +++ b/tests/test_wallet_subscription.py @@ -27,10 +27,8 @@ async def wallet(mint): @pytest.mark.asyncio +@pytest.mark.skipif(is_fake, reason="only regtest") async def test_wallet_subscription_mint(wallet: Wallet): - if is_fake: - settings.fakewallet_delay_outgoing_payment = 2 - if not wallet.mint_info.supports_nut(WEBSOCKETS_NUT): pytest.skip("No websocket support") From e2ac76d1a152f75a50420044cdf3babfd024c435 Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Wed, 19 Jun 2024 22:57:55 +0200 Subject: [PATCH 39/45] clean up --- cashu/lightning/fake.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/cashu/lightning/fake.py b/cashu/lightning/fake.py index b9eccbd1..7c50306c 100644 --- a/cashu/lightning/fake.py +++ b/cashu/lightning/fake.py @@ -165,14 +165,6 @@ async def pay_invoice(self, quote: MeltQuote, fee_limit: int) -> PaymentResponse ) async def get_invoice_status(self, checking_id: str) -> PaymentStatus: - # is_created_invoice = any( - # [checking_id == i.payment_hash for i in self.created_invoices] - # ) - # if not is_created_invoice: - # return PaymentStatus(paid=None) - # invoice = next( - # i for i in self.created_invoices if i.payment_hash == checking_id - # ) paid = False if settings.fakewallet_brr or ( settings.fakewallet_stochastic_invoice and random.random() > 0.7 @@ -191,12 +183,6 @@ async def get_invoice_status(self, checking_id: str) -> PaymentStatus: async def get_payment_status(self, _: str) -> PaymentStatus: return PaymentStatus(paid=settings.fakewallet_payment_state) - # async def get_invoice_quote(self, bolt11: str) -> InvoiceQuoteResponse: - # invoice_obj = decode(bolt11) - # assert invoice_obj.amount_msat, "invoice has no amount." - # amount = invoice_obj.amount_msat - # return InvoiceQuoteResponse(checking_id="", amount=amount) - async def get_payment_quote( self, melt_quote: PostMeltQuoteRequest ) -> PaymentQuoteResponse: From 917d46e7a78c354160ab9ab89bdcad58f339efcc Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Thu, 20 Jun 2024 11:53:32 +0200 Subject: [PATCH 40/45] tmp --- cashu/core/base.py | 2 +- cashu/mint/events/events.py | 31 ++++++++++--------------------- cashu/wallet/cli/cli.py | 6 +++--- 3 files changed, 14 insertions(+), 25 deletions(-) diff --git a/cashu/core/base.py b/cashu/core/base.py index 47cd63d6..549527f5 100644 --- a/cashu/core/base.py +++ b/cashu/core/base.py @@ -11,7 +11,7 @@ from cashu.core.json_rpc.base import JSONRPCSubscriptionKinds -from ..mint.events.events import LedgerEvent +from ..mint.events.ledger_event import LedgerEvent from .crypto.aes import AESCipher from .crypto.b_dhke import hash_to_curve from .crypto.keys import ( diff --git a/cashu/mint/events/events.py b/cashu/mint/events/events.py index 3e7db3fd..f3f5ee38 100644 --- a/cashu/mint/events/events.py +++ b/cashu/mint/events/events.py @@ -1,26 +1,9 @@ -from abc import ABC, abstractmethod - from loguru import logger -from pydantic import BaseModel -from ...core.json_rpc.base import JSONRPCSubscriptionKinds +from ...core.base import MeltQuote, MintQuote, ProofState +from ...core.models import PostMeltQuoteResponse, PostMintQuoteResponse from .client_manager import LedgerEventClientManager - - -class LedgerEvent(ABC, BaseModel): - """AbstractBaseClass for BaseModels that can be sent to the - LedgerEventManager for broadcasting subscription events to clients. - """ - - @property - @abstractmethod - def identifier(self) -> str: - pass - - @property - @abstractmethod - def kind(self) -> JSONRPCSubscriptionKinds: - pass +from .ledger_event import LedgerEvent class LedgerEventManager: @@ -46,7 +29,13 @@ def remove_client(self, client: LedgerEventClientManager) -> None: self.clients.remove(client) def serialize_event(self, event: LedgerEvent) -> dict: - return event.dict(exclude_unset=True, exclude_none=True) + if isinstance(event, MintQuote): + return_dict = PostMintQuoteResponse.parse_obj(event.dict()).dict() + elif isinstance(event, MeltQuote): + return_dict = PostMeltQuoteResponse.parse_obj(event.dict()).dict() + elif isinstance(event, ProofState): + return_dict = event.dict(exclude_unset=True, exclude_none=True) + return return_dict async def submit(self, event: LedgerEvent) -> None: if not isinstance(event, LedgerEvent): diff --git a/cashu/wallet/cli/cli.py b/cashu/wallet/cli/cli.py index 0f3a2259..3c600433 100644 --- a/cashu/wallet/cli/cli.py +++ b/cashu/wallet/cli/cli.py @@ -15,10 +15,11 @@ from click import Context from loguru import logger -from ...core.base import Invoice, Method, MintQuote, TokenV3, Unit +from ...core.base import Invoice, Method, TokenV3, Unit from ...core.helpers import sum_proofs from ...core.json_rpc.base import JSONRPCNotficationParams from ...core.logging import configure_logger +from ...core.models import PostMintQuoteResponse from ...core.settings import settings from ...nostr.client.client import NostrClient from ...tor.tor import TorProxy @@ -291,13 +292,12 @@ def mint_invoice_callback(msg: JSONRPCNotficationParams): if paid: return try: - quote = MintQuote.parse_obj(msg.payload) + quote = PostMintQuoteResponse.parse_obj(msg.payload) except Exception: return logger.debug(f"Received callback for quote: {quote}") if ( quote.paid - and not quote.issued and quote.request == invoice.bolt11 and msg.subId in subscription.callback_map.keys() ): From bbf08b6e40e8c2bb826675f1c022811b18cb6389 Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Tue, 25 Jun 2024 15:57:39 +0200 Subject: [PATCH 41/45] fix db issues and remove cached secrets --- .github/workflows/ci.yml | 2 - .github/workflows/tests.yml | 6 +- cashu/core/base.py | 2 +- cashu/core/settings.py | 1 - cashu/mint/db/read.py | 68 +++++++++ cashu/mint/db/write.py | 97 ++++++++++++ cashu/mint/events/__init__.py | 0 .../events/{client_manager.py => client.py} | 46 +++++- cashu/mint/events/event_model.py | 21 +++ cashu/mint/events/events.py | 23 ++- cashu/mint/ledger.py | 141 ++---------------- cashu/mint/router.py | 14 +- cashu/mint/router_deprecated.py | 2 +- cashu/mint/tasks.py | 2 +- cashu/mint/verification.py | 16 +- tests/test_mint_init.py | 22 +-- tests/test_mint_operations.py | 2 +- tests/test_mint_regtest.py | 4 +- tests/test_wallet_subscription.py | 31 ++-- 19 files changed, 310 insertions(+), 190 deletions(-) create mode 100644 cashu/mint/db/read.py create mode 100644 cashu/mint/db/write.py create mode 100644 cashu/mint/events/__init__.py rename cashu/mint/events/{client_manager.py => client.py} (76%) create mode 100644 cashu/mint/events/event_model.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fcae1ade..d035af3a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,6 @@ jobs: os: [ubuntu-latest] python-version: ["3.10"] poetry-version: ["1.7.1"] - mint-cache-secrets: ["false", "true"] mint-only-deprecated: ["false", "true"] mint-database: ["./test_data/test_mint", "postgres://cashu:cashu@localhost:5432/cashu"] backend-wallet-class: ["FakeWallet"] @@ -24,7 +23,6 @@ jobs: os: ${{ matrix.os }} python-version: ${{ matrix.python-version }} poetry-version: ${{ matrix.poetry-version }} - mint-cache-secrets: ${{ matrix.mint-cache-secrets }} mint-only-deprecated: ${{ matrix.mint-only-deprecated }} mint-database: ${{ matrix.mint-database }} regtest: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 42069702..40ecb60f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -15,16 +15,13 @@ on: os: default: "ubuntu-latest" type: string - mint-cache-secrets: - default: "false" - type: string mint-only-deprecated: default: "false" type: string jobs: poetry: - name: Run (mint-cache-secrets ${{ inputs.mint-cache-secrets }}, mint-only-deprecated ${{ inputs.mint-only-deprecated }}, mint-database ${{ inputs.mint-database }}) + name: Run (db ${{ inputs.mint-database }}, deprecated api ${{ inputs.mint-only-deprecated }}) runs-on: ${{ inputs.os }} services: postgres: @@ -54,7 +51,6 @@ jobs: MINT_HOST: localhost MINT_PORT: 3337 MINT_TEST_DATABASE: ${{ inputs.mint-database }} - MINT_CACHE_SECRETS: ${{ inputs.mint-cache-secrets }} DEBUG_MINT_ONLY_DEPRECATED: ${{ inputs.mint-only-deprecated }} TOR: false run: | diff --git a/cashu/core/base.py b/cashu/core/base.py index 549527f5..699cacea 100644 --- a/cashu/core/base.py +++ b/cashu/core/base.py @@ -11,7 +11,7 @@ from cashu.core.json_rpc.base import JSONRPCSubscriptionKinds -from ..mint.events.ledger_event import LedgerEvent +from ..mint.events.event_model import LedgerEvent from .crypto.aes import AESCipher from .crypto.b_dhke import hash_to_curve from .crypto.keys import ( diff --git a/cashu/core/settings.py b/cashu/core/settings.py index 23a8c4c4..b4c3269c 100644 --- a/cashu/core/settings.py +++ b/cashu/core/settings.py @@ -134,7 +134,6 @@ class FakeWalletSettings(MintSettings): fakewallet_delay_incoming_payment: Optional[int] = Field(default=3) fakewallet_stochastic_invoice: bool = Field(default=False) fakewallet_payment_state: Optional[bool] = Field(default=None) - mint_cache_secrets: bool = Field(default=True) class MintInformation(CashuSettings): diff --git a/cashu/mint/db/read.py b/cashu/mint/db/read.py new file mode 100644 index 00000000..b245b08f --- /dev/null +++ b/cashu/mint/db/read.py @@ -0,0 +1,68 @@ +from typing import Dict, List + +from ...core.base import Proof, ProofState, SpentState +from ...core.db import Database +from ..crud import LedgerCrud + + +class DbReadHelper: + db: Database + crud: LedgerCrud + + def __init__(self, db: Database, crud: LedgerCrud) -> None: + self.db = db + self.crud = crud + + async def _get_proofs_pending(self, Ys: List[str]) -> Dict[str, Proof]: + """Returns a dictionary of only those proofs that are pending. + The key is the Y=h2c(secret) and the value is the proof. + """ + proofs_pending = await self.crud.get_proofs_pending(Ys=Ys, db=self.db) + proofs_pending_dict = {p.Y: p for p in proofs_pending} + return proofs_pending_dict + + async def _get_proofs_spent(self, Ys: List[str]) -> Dict[str, Proof]: + """Returns a dictionary of all proofs that are spent. + The key is the Y=h2c(secret) and the value is the proof. + """ + proofs_spent_dict: Dict[str, Proof] = {} + # check used secrets in database + async with self.db.connect() as conn: + for Y in Ys: + spent_proof = await self.crud.get_proof_used(db=self.db, Y=Y, conn=conn) + if spent_proof: + proofs_spent_dict[Y] = spent_proof + return proofs_spent_dict + + async def get_proofs_states(self, Ys: List[str]) -> List[ProofState]: + """Checks if provided proofs are spend or are pending. + Used by wallets to check if their proofs have been redeemed by a receiver or they are still in-flight in a transaction. + + Returns two lists that are in the same order as the provided proofs. Wallet must match the list + to the proofs they have provided in order to figure out which proof is spendable or pending + and which isn't. + + Args: + Ys (List[str]): List of Y's of proofs to check + + Returns: + List[bool]: List of which proof is still spendable (True if still spendable, else False) + List[bool]: List of which proof are pending (True if pending, else False) + """ + states: List[ProofState] = [] + proofs_spent = await self._get_proofs_spent(Ys) + proofs_pending = await self._get_proofs_pending(Ys) + for Y in Ys: + if Y not in proofs_spent and Y not in proofs_pending: + states.append(ProofState(Y=Y, state=SpentState.unspent)) + elif Y not in proofs_spent and Y in proofs_pending: + states.append(ProofState(Y=Y, state=SpentState.pending)) + else: + states.append( + ProofState( + Y=Y, + state=SpentState.spent, + witness=proofs_spent[Y].witness, + ) + ) + return states diff --git a/cashu/mint/db/write.py b/cashu/mint/db/write.py new file mode 100644 index 00000000..2f307bf7 --- /dev/null +++ b/cashu/mint/db/write.py @@ -0,0 +1,97 @@ +import asyncio +from typing import List, Optional + +from loguru import logger + +from ...core.base import Proof, ProofState, SpentState +from ...core.db import Connection, Database, get_db_connection +from ...core.errors import ( + TransactionError, +) +from ..crud import LedgerCrud +from ..events.events import LedgerEventManager + + +class DbWriteHelper: + db: Database + crud: LedgerCrud + events: LedgerEventManager + proofs_pending_lock: asyncio.Lock = ( + asyncio.Lock() + ) # holds locks for proofs_pending database + + def __init__( + self, db: Database, crud: LedgerCrud, events: LedgerEventManager + ) -> None: + self.db = db + self.crud = crud + self.events = events + + async def _set_proofs_pending( + self, proofs: List[Proof], quote_id: Optional[str] = None + ) -> None: + """If none of the proofs is in the pending table (_validate_proofs_pending), adds proofs to + the list of pending proofs or removes them. Used as a mutex for proofs. + + Args: + proofs (List[Proof]): Proofs to add to pending table. + quote_id (Optional[str]): Melt quote ID. If it is not set, we assume the pending tokens to be from a swap. + + Raises: + Exception: At least one proof already in pending table. + """ + # first we check whether these proofs are pending already + async with self.proofs_pending_lock: + async with get_db_connection(self.db) as conn: + await self._validate_proofs_pending(proofs, conn) + try: + for p in proofs: + await self.crud.set_proof_pending( + proof=p, db=self.db, quote_id=quote_id, conn=conn + ) + await self.events.submit( + ProofState(Y=p.Y, state=SpentState.pending) + ) + except Exception as e: + logger.error(f"Failed to set proofs pending: {e}") + raise TransactionError("Failed to set proofs pending.") + + async def _unset_proofs_pending(self, proofs: List[Proof], spent=True) -> None: + """Deletes proofs from pending table. + + Args: + proofs (List[Proof]): Proofs to delete. + spent (bool): Whether the proofs have been spent or not. Defaults to True. + This should be False if the proofs were NOT invalidated before calling this function. + It is used to emit the unspent state for the proofs (otherwise the spent state is emitted + by the _invalidate_proofs function when the proofs are spent). + """ + async with self.proofs_pending_lock: + async with get_db_connection(self.db) as conn: + for p in proofs: + await self.crud.unset_proof_pending(proof=p, db=self.db, conn=conn) + if not spent: + await self.events.submit( + ProofState(Y=p.Y, state=SpentState.unspent) + ) + + async def _validate_proofs_pending( + self, proofs: List[Proof], conn: Optional[Connection] = None + ) -> None: + """Checks if any of the provided proofs is in the pending proofs table. + + Args: + proofs (List[Proof]): Proofs to check. + + Raises: + Exception: At least one of the proofs is in the pending table. + """ + if not ( + len( + await self.crud.get_proofs_pending( + Ys=[p.Y for p in proofs], db=self.db, conn=conn + ) + ) + == 0 + ): + raise TransactionError("proofs are pending.") diff --git a/cashu/mint/events/__init__.py b/cashu/mint/events/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cashu/mint/events/client_manager.py b/cashu/mint/events/client.py similarity index 76% rename from cashu/mint/events/client_manager.py rename to cashu/mint/events/client.py index dbf37f7d..29d74e32 100644 --- a/cashu/mint/events/client_manager.py +++ b/cashu/mint/events/client.py @@ -5,6 +5,8 @@ from fastapi import WebSocket from loguru import logger +from ...core.base import MeltQuote, MintQuote, ProofState +from ...core.db import Database from ...core.json_rpc.base import ( JSONRPCError, JSONRPCErrorCode, @@ -20,8 +22,12 @@ JSONRPCUnsubscribeParams, JSONRRPCSubscribeResponse, ) +from ...core.models import PostMeltQuoteResponse, PostMintQuoteResponse from ...core.settings import settings +from ..crud import LedgerCrud +from ..db.read import DbReadHelper from ..limit import limit_websocket +from .event_model import LedgerEvent class LedgerEventClientManager: @@ -30,10 +36,12 @@ class LedgerEventClientManager: JSONRPCSubscriptionKinds, dict[str, List[str]] ] = {} # [kind, [filter, List[subId]]] max_subscriptions = 1000 + db_read: DbReadHelper - def __init__(self, websocket: WebSocket): + def __init__(self, websocket: WebSocket, db: Database, crud: LedgerCrud): self.websocket = websocket self.subscriptions = {} + self.db_read = DbReadHelper(db, crud) async def start(self): await self.websocket.accept() @@ -162,7 +170,10 @@ async def _send_msg( await self.websocket.send_text(data.json()) def add_subscription( - self, kind: JSONRPCSubscriptionKinds, filters: List[str], subId: str + self, + kind: JSONRPCSubscriptionKinds, + filters: List[str], + subId: str, ) -> None: if kind not in self.subscriptions: self.subscriptions[kind] = {} @@ -175,6 +186,8 @@ def add_subscription( self.subscriptions[kind][filter] = [] logger.debug(f"Adding subscription {subId} for filter {filter}") self.subscriptions[kind][filter].append(subId) + # Initialize the subscription + asyncio.create_task(self._init_subscription(subId, filter, kind)) def remove_subscription(self, subId: str) -> None: for kind, sub_filters in self.subscriptions.items(): @@ -187,3 +200,32 @@ def remove_subscription(self, subId: str) -> None: self.subscriptions[kind][filter].remove(sub) return raise ValueError(f"Subscription not found: {subId}") + + def serialize_event(self, event: LedgerEvent) -> dict: + if isinstance(event, MintQuote): + return_dict = PostMintQuoteResponse.parse_obj(event.dict()).dict() + elif isinstance(event, MeltQuote): + return_dict = PostMeltQuoteResponse.parse_obj(event.dict()).dict() + elif isinstance(event, ProofState): + return_dict = event.dict(exclude_unset=True, exclude_none=True) + return return_dict + + async def _init_subscription( + self, subId: str, filter: str, kind: JSONRPCSubscriptionKinds + ): + if kind == JSONRPCSubscriptionKinds.BOLT11_MINT_QUOTE: + mint_quote = await self.db_read.crud.get_mint_quote( + quote_id=filter, db=self.db_read.db + ) + if mint_quote: + await self._send_obj(mint_quote.dict(), subId) + elif kind == JSONRPCSubscriptionKinds.BOLT11_MELT_QUOTE: + melt_quote = await self.db_read.crud.get_melt_quote( + quote_id=filter, db=self.db_read.db + ) + if melt_quote: + await self._send_obj(melt_quote.dict(), subId) + elif kind == JSONRPCSubscriptionKinds.PROOF_STATE: + proofs = await self.db_read.get_proofs_states(Ys=[filter]) + if len(proofs): + await self._send_obj(proofs[0].dict(), subId) diff --git a/cashu/mint/events/event_model.py b/cashu/mint/events/event_model.py new file mode 100644 index 00000000..a6220b4a --- /dev/null +++ b/cashu/mint/events/event_model.py @@ -0,0 +1,21 @@ +from abc import ABC, abstractmethod + +from pydantic import BaseModel + +from ...core.json_rpc.base import JSONRPCSubscriptionKinds + + +class LedgerEvent(ABC, BaseModel): + """AbstractBaseClass for BaseModels that can be sent to the + LedgerEventManager for broadcasting subscription events to clients. + """ + + @property + @abstractmethod + def identifier(self) -> str: + pass + + @property + @abstractmethod + def kind(self) -> JSONRPCSubscriptionKinds: + pass diff --git a/cashu/mint/events/events.py b/cashu/mint/events/events.py index f3f5ee38..10b40a9e 100644 --- a/cashu/mint/events/events.py +++ b/cashu/mint/events/events.py @@ -1,9 +1,14 @@ +import asyncio + +from fastapi import WebSocket from loguru import logger from ...core.base import MeltQuote, MintQuote, ProofState +from ...core.db import Database from ...core.models import PostMeltQuoteResponse, PostMintQuoteResponse -from .client_manager import LedgerEventClientManager -from .ledger_event import LedgerEvent +from ..crud import LedgerCrud +from .client import LedgerEventClientManager +from .event_model import LedgerEvent class LedgerEventManager: @@ -18,12 +23,15 @@ class LedgerEventManager: MAX_CLIENTS = 1000 - def add_client(self, client: LedgerEventClientManager) -> bool: + def add_client( + self, websocket: WebSocket, db: Database, crud: LedgerCrud + ) -> LedgerEventClientManager: + client = LedgerEventClientManager(websocket, db, crud) if len(self.clients) >= self.MAX_CLIENTS: - return False + raise Exception("too many clients") self.clients.append(client) logger.debug(f"Added websocket subscription client {client}") - return True + return client def remove_client(self, client: LedgerEventClientManager) -> None: self.clients.remove(client) @@ -41,10 +49,13 @@ async def submit(self, event: LedgerEvent) -> None: if not isinstance(event, LedgerEvent): raise ValueError(f"Unsupported event object type {type(event)}") + # check if any clients are subscribed to this event for client in self.clients: kind_sub = client.subscriptions.get(event.kind, {}) for sub in kind_sub.get(event.identifier, []): logger.trace( f"Submitting event to sub {sub}: {self.serialize_event(event)}" ) - await client._send_obj(self.serialize_event(event), subId=sub) + asyncio.create_task( + client._send_obj(self.serialize_event(event), subId=sub) + ) diff --git a/cashu/mint/ledger.py b/cashu/mint/ledger.py index db5639cb..37a23cec 100644 --- a/cashu/mint/ledger.py +++ b/cashu/mint/ledger.py @@ -52,6 +52,8 @@ ) from ..mint.crud import LedgerCrudSqlite from .conditions import LedgerSpendingConditions +from .db.read import DbReadHelper +from .db.write import DbWriteHelper from .events.events import LedgerEventManager from .features import LedgerFeatures from .tasks import LedgerTasks @@ -61,11 +63,9 @@ class Ledger(LedgerVerification, LedgerSpendingConditions, LedgerTasks, LedgerFeatures): backends: Mapping[Method, Mapping[Unit, LightningBackend]] = {} locks: Dict[str, asyncio.Lock] = {} # holds multiprocessing locks - proofs_pending_lock: asyncio.Lock = ( - asyncio.Lock() - ) # holds locks for proofs_pending database keysets: Dict[str, MintKeyset] = {} events = LedgerEventManager() + db_read: DbReadHelper def __init__( self, @@ -96,7 +96,8 @@ def __init__( self.crud = crud self.backends = backends self.pubkey = derive_pubkey(self.seed) - self.spent_proofs: Dict[str, Proof] = {} + self.db_read = DbReadHelper(self.db, self.crud) + self.db_write = DbWriteHelper(self.db, self.crud, self.events) # ------- STARTUP ------- @@ -106,8 +107,6 @@ async def startup_ledger(self): await self.dispatch_listeners() async def _startup_ledger(self): - if settings.mint_cache_secrets: - await self.load_used_proofs() await self.init_keysets() for derivation_path in settings.mint_derivation_path_list: @@ -163,12 +162,12 @@ async def _check_pending_proofs_and_melt_quotes(self): proofs=pending_proofs, quote_id=quote.quote ) # unset pending - await self._unset_proofs_pending(pending_proofs) + await self.db_write._unset_proofs_pending(pending_proofs) elif payment.failed: logger.info(f"Melt quote {quote.quote} state: failed") # unset pending - await self._unset_proofs_pending(pending_proofs, spent=False) + await self.db_write._unset_proofs_pending(pending_proofs, spent=False) elif payment.pending: logger.info(f"Melt quote {quote.quote} state: pending") pass @@ -296,7 +295,6 @@ async def _invalidate_proofs( proofs (List[Proof]): Proofs to add to known secret table. conn: (Optional[Connection], optional): Database connection to reuse. Will create a new one if not given. Defaults to None. """ - self.spent_proofs.update({p.Y: p for p in proofs}) async with get_db_connection(self.db, conn) as conn: # store in db for p in proofs: @@ -787,7 +785,7 @@ async def melt_mint_settle_internally( mint_quote.paid = True mint_quote.paid_time = melt_quote.paid_time - async with self.db.connect() as conn: + async with get_db_connection(self.db) as conn: await self.crud.update_melt_quote(quote=melt_quote, db=self.db, conn=conn) await self.crud.update_mint_quote(quote=mint_quote, db=self.db, conn=conn) @@ -863,7 +861,7 @@ async def melt( await self.verify_inputs_and_outputs(proofs=proofs) # set proofs to pending to avoid race conditions - await self._set_proofs_pending(proofs, quote_id=melt_quote.quote) + await self.db_write._set_proofs_pending(proofs, quote_id=melt_quote.quote) try: # settle the transaction internally if there is a mint quote with the same payment request melt_quote = await self.melt_mint_settle_internally(melt_quote, proofs) @@ -911,7 +909,7 @@ async def melt( raise e finally: # delete proofs from pending list - await self._unset_proofs_pending(proofs) + await self.db_write._unset_proofs_pending(proofs) return melt_quote.proof or "", return_promises @@ -945,7 +943,7 @@ async def split( # verify spending inputs, outputs, and spending conditions await self.verify_inputs_and_outputs(proofs=proofs, outputs=outputs) - await self._set_proofs_pending(proofs) + await self.db_write._set_proofs_pending(proofs) try: # Mark proofs as used and prepare new promises async with get_db_connection(self.db) as conn: @@ -958,7 +956,7 @@ async def split( raise e finally: # delete proofs from pending list - await self._unset_proofs_pending(proofs) + await self.db_write._unset_proofs_pending(proofs) logger.trace("split successful") return promises @@ -968,7 +966,7 @@ async def restore( ) -> Tuple[List[BlindedMessage], List[BlindedSignature]]: signatures: List[BlindedSignature] = [] return_outputs: List[BlindedMessage] = [] - async with self.db.connect() as conn: + async with get_db_connection(self.db) as conn: for output in outputs: logger.trace(f"looking for promise: {output}") promise = await self.crud.get_promise( @@ -1047,116 +1045,3 @@ async def _generate_promises( ) signatures.append(signature) return signatures - - # ------- PROOFS ------- - - async def load_used_proofs(self) -> None: - """Load all used proofs from database.""" - if not settings.mint_cache_secrets: - raise Exception("MINT_CACHE_SECRETS must be set to TRUE") - logger.debug("Loading used proofs into memory") - spent_proofs_list = await self.crud.get_spent_proofs(db=self.db) or [] - logger.debug(f"Loaded {len(spent_proofs_list)} used proofs") - self.spent_proofs = {p.Y: p for p in spent_proofs_list} - - async def check_proofs_state(self, Ys: List[str]) -> List[ProofState]: - """Checks if provided proofs are spend or are pending. - Used by wallets to check if their proofs have been redeemed by a receiver or they are still in-flight in a transaction. - - Returns two lists that are in the same order as the provided proofs. Wallet must match the list - to the proofs they have provided in order to figure out which proof is spendable or pending - and which isn't. - - Args: - Ys (List[str]): List of Y's of proofs to check - - Returns: - List[bool]: List of which proof is still spendable (True if still spendable, else False) - List[bool]: List of which proof are pending (True if pending, else False) - """ - states: List[ProofState] = [] - proofs_spent = await self._get_proofs_spent(Ys) - proofs_pending = await self._get_proofs_pending(Ys) - for Y in Ys: - if Y not in proofs_spent and Y not in proofs_pending: - states.append(ProofState(Y=Y, state=SpentState.unspent)) - elif Y not in proofs_spent and Y in proofs_pending: - states.append(ProofState(Y=Y, state=SpentState.pending)) - else: - states.append( - ProofState( - Y=Y, - state=SpentState.spent, - witness=proofs_spent[Y].witness, - ) - ) - return states - - async def _set_proofs_pending( - self, proofs: List[Proof], quote_id: Optional[str] = None - ) -> None: - """If none of the proofs is in the pending table (_validate_proofs_pending), adds proofs to - the list of pending proofs or removes them. Used as a mutex for proofs. - - Args: - proofs (List[Proof]): Proofs to add to pending table. - quote_id (Optional[str]): Melt quote ID. If it is not set, we assume the pending tokens to be from a swap. - - Raises: - Exception: At least one proof already in pending table. - """ - # first we check whether these proofs are pending already - async with self.proofs_pending_lock: - async with self.db.connect() as conn: - await self._validate_proofs_pending(proofs, conn) - try: - for p in proofs: - await self.crud.set_proof_pending( - proof=p, db=self.db, quote_id=quote_id, conn=conn - ) - await self.events.submit( - ProofState(Y=p.Y, state=SpentState.pending) - ) - except Exception as e: - logger.error(f"Failed to set proofs pending: {e}") - raise TransactionError("Failed to set proofs pending.") - - async def _unset_proofs_pending(self, proofs: List[Proof], spent=True) -> None: - """Deletes proofs from pending table. - - Args: - proofs (List[Proof]): Proofs to delete. - spent (bool): Whether the proofs have been spent or not. Defaults to True. - This should be False if the proofs were NOT invalidated before calling this function. - It is used to emit the unspent state for the proofs (otherwise the spent state is emitted - by the _invalidate_proofs function when the proofs are spent). - """ - async with self.proofs_pending_lock: - async with self.db.connect() as conn: - for p in proofs: - await self.crud.unset_proof_pending(proof=p, db=self.db, conn=conn) - if not spent: - await self.events.submit( - ProofState(Y=p.Y, state=SpentState.unspent) - ) - - async def _validate_proofs_pending( - self, proofs: List[Proof], conn: Optional[Connection] = None - ) -> None: - """Checks if any of the provided proofs is in the pending proofs table. - - Args: - proofs (List[Proof]): Proofs to check. - - Raises: - Exception: At least one of the proofs is in the pending table. - """ - if not ( - len( - await self.crud.get_proofs_pending( - Ys=[p.Y for p in proofs], db=self.db, conn=conn - ) - ) - == 0 - ): - raise TransactionError("proofs are pending.") diff --git a/cashu/mint/router.py b/cashu/mint/router.py index 13702713..1bc42167 100644 --- a/cashu/mint/router.py +++ b/cashu/mint/router.py @@ -3,8 +3,6 @@ from fastapi import APIRouter, Request, WebSocket from loguru import logger -from cashu.mint.events.client_manager import LedgerEventClientManager - from ..core.errors import KeysetNotFoundError from ..core.models import ( GetInfoResponse, @@ -195,16 +193,20 @@ async def get_mint_quote(request: Request, quote: str) -> PostMintQuoteResponse: @router.websocket("/v1/ws", name="Websocket endpoint for subscriptions") async def websocket_endpoint(websocket: WebSocket): limit_websocket(websocket) - client = LedgerEventClientManager(websocket=websocket) - success = ledger.events.add_client(client) - if not success: + try: + client = ledger.events.add_client(websocket, ledger.db, ledger.crud) + except Exception as e: + logger.debug(f"Exception: {e}") await asyncio.wait_for(websocket.close(), timeout=1) return + try: + # this will block until the session is closed await client.start() except Exception as e: logger.debug(f"Exception: {e}") ledger.events.remove_client(client) + finally: await asyncio.wait_for(websocket.close(), timeout=1) @@ -350,7 +352,7 @@ async def check_state( ) -> PostCheckStateResponse: """Check whether a secret has been spent already or not.""" logger.trace(f"> POST /v1/checkstate: {payload}") - proof_states = await ledger.check_proofs_state(payload.Ys) + proof_states = await ledger.db_read.get_proofs_states(payload.Ys) return PostCheckStateResponse(states=proof_states) diff --git a/cashu/mint/router_deprecated.py b/cashu/mint/router_deprecated.py index 67976d2a..520ec8d7 100644 --- a/cashu/mint/router_deprecated.py +++ b/cashu/mint/router_deprecated.py @@ -341,7 +341,7 @@ async def check_spendable_deprecated( ) -> CheckSpendableResponse_deprecated: """Check whether a secret has been spent already or not.""" logger.trace(f"> POST /check: {payload}") - proofs_state = await ledger.check_proofs_state([p.Y for p in payload.proofs]) + proofs_state = await ledger.db_read.get_proofs_states([p.Y for p in payload.proofs]) spendableList: List[bool] = [] pendingList: List[bool] = [] for proof_state in proofs_state: diff --git a/cashu/mint/tasks.py b/cashu/mint/tasks.py index bfa49bb8..413421ce 100644 --- a/cashu/mint/tasks.py +++ b/cashu/mint/tasks.py @@ -30,7 +30,7 @@ async def invoice_listener(self, backend: LightningBackend) -> None: await self.invoice_callback_dispatcher(checking_id) async def invoice_callback_dispatcher(self, checking_id: str) -> None: - logger.success(f"invoice callback dispatcher: {checking_id}") + logger.debug(f"Invoice callback dispatcher: {checking_id}") # TODO: Explicitly check for the quote payment state before setting it as paid # db read, quote.paid = True, db write should be refactored and moved to ledger.py quote = await self.crud.get_mint_quote(checking_id=checking_id, db=self.db) diff --git a/cashu/mint/verification.py b/cashu/mint/verification.py index 62275165..006a5daf 100644 --- a/cashu/mint/verification.py +++ b/cashu/mint/verification.py @@ -35,7 +35,6 @@ class LedgerVerification( keyset: MintKeyset keysets: Dict[str, MintKeyset] - spent_proofs: Dict[str, Proof] crud: LedgerCrud db: Database lightning: Dict[Unit, LightningBackend] @@ -167,21 +166,12 @@ async def _get_proofs_spent(self, Ys: List[str]) -> Dict[str, Proof]: The key is the Y=h2c(secret) and the value is the proof. """ proofs_spent_dict: Dict[str, Proof] = {} - if settings.mint_cache_secrets: - # check used secrets in memory + # check used secrets in database + async with self.db.connect() as conn: for Y in Ys: - spent_proof = self.spent_proofs.get(Y) + spent_proof = await self.crud.get_proof_used(db=self.db, Y=Y, conn=conn) if spent_proof: proofs_spent_dict[Y] = spent_proof - else: - # check used secrets in database - async with self.db.connect() as conn: - for Y in Ys: - spent_proof = await self.crud.get_proof_used( - db=self.db, Y=Y, conn=conn - ) - if spent_proof: - proofs_spent_dict[Y] = spent_proof return proofs_spent_dict def _verify_secret_criteria(self, proof: Proof) -> Literal[True]: diff --git a/tests/test_mint_init.py b/tests/test_mint_init.py index 3b3c75c8..de8cedd9 100644 --- a/tests/test_mint_init.py +++ b/tests/test_mint_init.py @@ -171,7 +171,7 @@ async def test_startup_fakewallet_pending_quote_success(ledger: Ledger): """Startup routine test. Expects that a pending proofs are removed form the pending db after the startup routine determines that the associated melt quote was paid.""" pending_proof, quote = await create_pending_melts(ledger) - states = await ledger.check_proofs_state([pending_proof.Y]) + states = await ledger.db_read.get_proofs_states([pending_proof.Y]) assert states[0].state == SpentState.pending settings.fakewallet_payment_state = True # run startup routinge @@ -184,7 +184,7 @@ async def test_startup_fakewallet_pending_quote_success(ledger: Ledger): assert not melt_quotes # expect that proofs are spent - states = await ledger.check_proofs_state([pending_proof.Y]) + states = await ledger.db_read.get_proofs_states([pending_proof.Y]) assert states[0].state == SpentState.spent @@ -197,7 +197,7 @@ async def test_startup_fakewallet_pending_quote_failure(ledger: Ledger): The failure is simulated by setting the fakewallet_payment_state to False. """ pending_proof, quote = await create_pending_melts(ledger) - states = await ledger.check_proofs_state([pending_proof.Y]) + states = await ledger.db_read.get_proofs_states([pending_proof.Y]) assert states[0].state == SpentState.pending settings.fakewallet_payment_state = False # run startup routinge @@ -210,7 +210,7 @@ async def test_startup_fakewallet_pending_quote_failure(ledger: Ledger): assert not melt_quotes # expect that proofs are unspent - states = await ledger.check_proofs_state([pending_proof.Y]) + states = await ledger.db_read.get_proofs_states([pending_proof.Y]) assert states[0].state == SpentState.unspent @@ -218,7 +218,7 @@ async def test_startup_fakewallet_pending_quote_failure(ledger: Ledger): @pytest.mark.skipif(is_regtest, reason="only for fake wallet") async def test_startup_fakewallet_pending_quote_pending(ledger: Ledger): pending_proof, quote = await create_pending_melts(ledger) - states = await ledger.check_proofs_state([pending_proof.Y]) + states = await ledger.db_read.get_proofs_states([pending_proof.Y]) assert states[0].state == SpentState.pending settings.fakewallet_payment_state = None # run startup routinge @@ -231,7 +231,7 @@ async def test_startup_fakewallet_pending_quote_pending(ledger: Ledger): assert melt_quotes # expect that proofs are still pending - states = await ledger.check_proofs_state([pending_proof.Y]) + states = await ledger.db_read.get_proofs_states([pending_proof.Y]) assert states[0].state == SpentState.pending @@ -273,7 +273,7 @@ async def test_startup_regtest_pending_quote_pending(wallet: Wallet, ledger: Led assert melt_quotes # expect that proofs are still pending - states = await ledger.check_proofs_state([p.Y for p in send_proofs]) + states = await ledger.db_read.get_proofs_states([p.Y for p in send_proofs]) assert all([s.state == SpentState.pending for s in states]) # only now settle the invoice @@ -307,7 +307,7 @@ async def test_startup_regtest_pending_quote_success(wallet: Wallet, ledger: Led ) await asyncio.sleep(SLEEP_TIME) # expect that proofs are pending - states = await ledger.check_proofs_state([p.Y for p in send_proofs]) + states = await ledger.db_read.get_proofs_states([p.Y for p in send_proofs]) assert all([s.state == SpentState.pending for s in states]) settle_invoice(preimage=preimage) @@ -323,7 +323,7 @@ async def test_startup_regtest_pending_quote_success(wallet: Wallet, ledger: Led assert not melt_quotes # expect that proofs are spent - states = await ledger.check_proofs_state([p.Y for p in send_proofs]) + states = await ledger.db_read.get_proofs_states([p.Y for p in send_proofs]) assert all([s.state == SpentState.spent for s in states]) @@ -358,7 +358,7 @@ async def test_startup_regtest_pending_quote_failure(wallet: Wallet, ledger: Led await asyncio.sleep(SLEEP_TIME) # expect that proofs are pending - states = await ledger.check_proofs_state([p.Y for p in send_proofs]) + states = await ledger.db_read.get_proofs_states([p.Y for p in send_proofs]) assert all([s.state == SpentState.pending for s in states]) cancel_invoice(preimage_hash=preimage_hash) @@ -374,5 +374,5 @@ async def test_startup_regtest_pending_quote_failure(wallet: Wallet, ledger: Led assert not melt_quotes # expect that proofs are unspent - states = await ledger.check_proofs_state([p.Y for p in send_proofs]) + states = await ledger.db_read.get_proofs_states([p.Y for p in send_proofs]) assert all([s.state == SpentState.unspent for s in states]) diff --git a/tests/test_mint_operations.py b/tests/test_mint_operations.py index e1206c09..2681aa55 100644 --- a/tests/test_mint_operations.py +++ b/tests/test_mint_operations.py @@ -372,7 +372,7 @@ async def test_check_proof_state(wallet1: Wallet, ledger: Ledger): keep_proofs, send_proofs = await wallet1.split_to_send(wallet1.proofs, 10) - proof_states = await ledger.check_proofs_state(Ys=[p.Y for p in send_proofs]) + proof_states = await ledger.db_read.get_proofs_states(Ys=[p.Y for p in send_proofs]) assert all([p.state.value == "UNSPENT" for p in proof_states]) diff --git a/tests/test_mint_regtest.py b/tests/test_mint_regtest.py index e065eaac..bb5eccee 100644 --- a/tests/test_mint_regtest.py +++ b/tests/test_mint_regtest.py @@ -62,7 +62,7 @@ async def test_regtest_pending_quote(wallet: Wallet, ledger: Ledger): assert melt_quotes # expect that proofs are still pending - states = await ledger.check_proofs_state([p.Y for p in send_proofs]) + states = await ledger.db_read.get_proofs_states([p.Y for p in send_proofs]) assert all([s.state == SpentState.pending for s in states]) # only now settle the invoice @@ -70,7 +70,7 @@ async def test_regtest_pending_quote(wallet: Wallet, ledger: Ledger): await asyncio.sleep(SLEEP_TIME) # expect that proofs are now spent - states = await ledger.check_proofs_state([p.Y for p in send_proofs]) + states = await ledger.db_read.get_proofs_states([p.Y for p in send_proofs]) assert all([s.state == SpentState.spent for s in states]) # expect that no melt quote is pending diff --git a/tests/test_wallet_subscription.py b/tests/test_wallet_subscription.py index 60c512bb..1f024bab 100644 --- a/tests/test_wallet_subscription.py +++ b/tests/test_wallet_subscription.py @@ -57,14 +57,18 @@ def callback(msg: JSONRPCNotficationParams): assert triggered # assert len(msg_stack) == 1 - assert msg_stack[0].payload["paid"] is True - assert msg_stack[0].payload["issued"] is False + + assert msg_stack[1].payload["paid"] is False + assert msg_stack[1].payload["issued"] is False + + assert msg_stack[1].payload["paid"] is True + assert msg_stack[1].payload["issued"] is False # await asyncio.sleep(2) # this will cause a second message - assert len(msg_stack) == 2 - assert msg_stack[1].payload["paid"] is True - assert msg_stack[1].payload["issued"] is True + assert len(msg_stack) == 3 + assert msg_stack[2].payload["paid"] is True + assert msg_stack[2].payload["issued"] is True @pytest.mark.asyncio @@ -95,17 +99,24 @@ def callback(msg: JSONRPCNotficationParams): await asyncio.sleep(wait) assert triggered - # we receive 2 messages for each subscription - assert len(msg_stack) == n_subscriptions * 2 + # we receive 3 messages for each subscription: + # initial state (UNSPENT), pending state (PENDING), spent state (SPENT) + assert len(msg_stack) == n_subscriptions * 3 - # the first one is the PENDING state + # the first one is the UNSPENT state pending_stack = msg_stack[:n_subscriptions] for msg in pending_stack: + proof_state = ProofState.parse_obj(msg.payload) + assert proof_state.state.value == "UNSPENT" + + # the second one is the PENDING state + spent_stack = msg_stack[n_subscriptions : n_subscriptions * 2] + for msg in spent_stack: proof_state = ProofState.parse_obj(msg.payload) assert proof_state.state.value == "PENDING" - # the second one is the SPENT state - spent_stack = msg_stack[n_subscriptions:] + # the third one is the SPENT state + spent_stack = msg_stack[n_subscriptions * 2 :] for msg in spent_stack: proof_state = ProofState.parse_obj(msg.payload) assert proof_state.state.value == "SPENT" From 083d427fe037be67d6fc6e42c083809a5538a8c2 Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Tue, 25 Jun 2024 17:53:41 +0200 Subject: [PATCH 42/45] fix tests --- cashu/mint/events/client.py | 4 ++-- tests/test_wallet_subscription.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cashu/mint/events/client.py b/cashu/mint/events/client.py index 29d74e32..e54c0736 100644 --- a/cashu/mint/events/client.py +++ b/cashu/mint/events/client.py @@ -118,7 +118,7 @@ async def start(self): # Handle the request try: - logger.debug(f"Request: {req}") + logger.debug(f"Request: {req.json()}") resp = await self._handle_request(req) # Send the response await self._send_msg(resp) @@ -166,7 +166,7 @@ async def _send_obj(self, data: dict, subId: str): async def _send_msg( self, data: Union[JSONRPCResponse, JSONRPCNotification, JSONRPCErrorResponse] ): - logger.debug(f"Sending websocket message: {data}") + logger.debug(f"Sending websocket message: {data.json()}") await self.websocket.send_text(data.json()) def add_subscription( diff --git a/tests/test_wallet_subscription.py b/tests/test_wallet_subscription.py index 1f024bab..271e47eb 100644 --- a/tests/test_wallet_subscription.py +++ b/tests/test_wallet_subscription.py @@ -58,8 +58,8 @@ def callback(msg: JSONRPCNotficationParams): assert triggered # assert len(msg_stack) == 1 - assert msg_stack[1].payload["paid"] is False - assert msg_stack[1].payload["issued"] is False + assert msg_stack[0].payload["paid"] is False + assert msg_stack[0].payload["issued"] is False assert msg_stack[1].payload["paid"] is True assert msg_stack[1].payload["issued"] is False From 20fa96a6914be7958aca539c22eb618236ee299a Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Tue, 25 Jun 2024 18:52:13 +0200 Subject: [PATCH 43/45] blindly try pipeline --- tests/test_wallet_subscription.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/test_wallet_subscription.py b/tests/test_wallet_subscription.py index 271e47eb..8d84d19a 100644 --- a/tests/test_wallet_subscription.py +++ b/tests/test_wallet_subscription.py @@ -56,19 +56,15 @@ def callback(msg: JSONRPCNotficationParams): # first we expect the issued=False state to arrive assert triggered - # assert len(msg_stack) == 1 + assert len(msg_stack) == 3 assert msg_stack[0].payload["paid"] is False - assert msg_stack[0].payload["issued"] is False assert msg_stack[1].payload["paid"] is True - assert msg_stack[1].payload["issued"] is False # await asyncio.sleep(2) # this will cause a second message - assert len(msg_stack) == 3 assert msg_stack[2].payload["paid"] is True - assert msg_stack[2].payload["issued"] is True @pytest.mark.asyncio From 50c85da50cd0e3a12663b1e5aa589e2214c1300f Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Tue, 25 Jun 2024 18:57:24 +0200 Subject: [PATCH 44/45] remove comments --- tests/test_wallet_subscription.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_wallet_subscription.py b/tests/test_wallet_subscription.py index 8d84d19a..6533296b 100644 --- a/tests/test_wallet_subscription.py +++ b/tests/test_wallet_subscription.py @@ -60,10 +60,9 @@ def callback(msg: JSONRPCNotficationParams): assert msg_stack[0].payload["paid"] is False + # TODO: we also send this when the quote is pending but the API does not express that yet assert msg_stack[1].payload["paid"] is True - # await asyncio.sleep(2) - # this will cause a second message assert msg_stack[2].payload["paid"] is True From 65dd5af2cba21bc4798469c5ca86f30258aa0e17 Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Tue, 25 Jun 2024 18:58:31 +0200 Subject: [PATCH 45/45] comments --- tests/test_wallet_subscription.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_wallet_subscription.py b/tests/test_wallet_subscription.py index 6533296b..d81896da 100644 --- a/tests/test_wallet_subscription.py +++ b/tests/test_wallet_subscription.py @@ -52,6 +52,8 @@ def callback(msg: JSONRPCNotficationParams): await asyncio.sleep(wait + 2) # TODO: check for pending and paid states according to: https://github.com/cashubtc/nuts/pull/136 + # TODO: we have three messages here, but the value "paid" only changes once + # the mint sends an update when the quote is pending but the API does not express that yet # first we expect the issued=False state to arrive @@ -60,7 +62,6 @@ def callback(msg: JSONRPCNotficationParams): assert msg_stack[0].payload["paid"] is False - # TODO: we also send this when the quote is pending but the API does not express that yet assert msg_stack[1].payload["paid"] is True assert msg_stack[2].payload["paid"] is True