diff --git a/.github/workflows/kalatori-test.yml b/.github/workflows/kalatori-test.yml index 61de1d9..0ed421d 100644 --- a/.github/workflows/kalatori-test.yml +++ b/.github/workflows/kalatori-test.yml @@ -46,6 +46,11 @@ jobs: run: | docker-compose build + - name: Create Network + working-directory: ./tests + run: | + docker network create kalatori-network + - name: Run Tests working-directory: ./tests run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 9275c74..7265f90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ All notable changes to this project will be documented in this file. +## [0.3] - 2024-11-28 + +This is a public beta release of the Kalatori daemon. While it adheres to the [API specs](https://alzymologist.github.io/kalatori-api), it is still under active development. We encourage you to test it and provide feedback. + + ## [0.2.8] - 2024-11-13 ### 🚀 Features diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0066569..3422a98 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,3 +1,13 @@ +## Issues + +You can find issues waiting to be solved or create a new one in the [issues](https://github.com/Alzymologist/Kalatori-backend/issues) section. + +## Prerequisites + +- Rust: stable version +- Docker: to run tests and spawn chopsticks instances +- Node.js and Yarn: to run tests + ## Preparing development environment It's possible to mimic to spawn chopsticks instances in parallel for development purposes. @@ -5,8 +15,9 @@ Chopsticks Dockerfile exposes 4 ports (8000, 8500, 9000, 9500), so you can spawn Note that the RPCs are not real, so the changes made on one chopsticks instance will not affect the others. 1. `cd chopsticks` -2. `docker compose up`, in case you want to just 2 instances edit the docker-compose.yml file -3. start the app with `KALATORI_CONFIG` environment variable pointing to `configs/chopsticks.toml` +2. Create docker network (do once) `docker network create kalatori-network` +3. `docker compose up`, in case you want to just 2 instances edit the docker-compose.yml file +4. start the app with `KALATORI_CONFIG` environment variable pointing to `configs/chopsticks.toml` ## Running tests locally diff --git a/Cargo.lock b/Cargo.lock index 997bcb1..e825795 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1622,7 +1622,7 @@ dependencies = [ [[package]] name = "kalatori" -version = "0.2.8" +version = "0.3.0" dependencies = [ "ahash", "async-lock", @@ -1788,7 +1788,8 @@ dependencies = [ [[package]] name = "mnemonic-external" version = "0.1.0" -source = "git+https://github.com/Alzymologist/mnemonic-external#2a02a0795e1532b1115e8d13e084c90539f31752" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acd8b607d24e1721a79eb4e30054fe776bf3956a87bd57f59c15fa1f72ad8721" dependencies = [ "sha2", "zeroize", @@ -2878,8 +2879,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "substrate-constructor" -version = "0.1.0" -source = "git+https://github.com/Alzymologist/substrate-constructor#9e4f0b27596658a9ecd3fd5bf5423713a911de54" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "269619703d949df9b9a1772ed2cdc6b2ff805dd9ff7e0f5c19241bc53cff6d93" dependencies = [ "bitvec", "external-memory-tools", @@ -2898,7 +2900,8 @@ dependencies = [ [[package]] name = "substrate-crypto-light" version = "0.1.0" -source = "git+https://github.com/Alzymologist/substrate-crypto-light#e1c07e19ccb19862accc2771d3b7ec0f86c80a36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86eaa081c10ed391947506525a50da570841ccf93f83a642ff91f36732e91b51" dependencies = [ "base58", "blake2b_simd", @@ -2917,8 +2920,9 @@ dependencies = [ [[package]] name = "substrate_parser" -version = "0.6.1" -source = "git+https://github.com/Alzymologist/substrate-parser#3a4a2c51d3d0c0141f1942631846f57b5bfe8b07" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7608864a21e2c356727f7ac0206252fb2c1cc4cee7c8539216528c6b54efca5" dependencies = [ "bitvec", "external-memory-tools", diff --git a/Cargo.toml b/Cargo.toml index dc8cc66..e931f85 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,14 +1,21 @@ [package] name = "kalatori" -authors = ["Alzymologist Oy "] -version = "0.2.8" +authors = [ + "Alexander Slesarev ", + "Vova Lando ", + "Artyom Sheviakov ", +] +version = "0.3.0" edition = "2021" description = "A gateway daemon for Kalatori." license = "GPL-3.0-or-later" +homepage = "https://github.com/Alzymologist/Kalatori-backend" repository = "https://github.com/Alzymologist/Kalatori-backend" -readme = true +documentation = "README.md" +readme = "README.md" keywords = ["substrate", "blockchain", "finance", "service", "middleware"] categories = ["finance"] +exclude = [".github", "cliff.toml", "Dockerfile", "chopsticks/", "configs", "is_version_greater.sh", "shoot.sh", "start.sh", "/tests"] rust-version = "1.82" [dependencies] @@ -61,10 +68,10 @@ async-lock = "3" time = "0.3" reqwest = "0.12" -substrate_parser = { git = "https://github.com/Alzymologist/substrate-parser" } -substrate-constructor = { git = "https://github.com/Alzymologist/substrate-constructor" } -mnemonic-external = { git = "https://github.com/Alzymologist/mnemonic-external" } -substrate-crypto-light = { git = "https://github.com/Alzymologist/substrate-crypto-light" } +substrate_parser = "0.7.0" +substrate-constructor = "0.2.0" +mnemonic-external = "0.1.0" +substrate-crypto-light = "0.1.0" [build-dependencies] # Don't forget to update me in `[dependencies]`! diff --git a/README.md b/README.md index db80f50..5ddd072 100644 --- a/README.md +++ b/README.md @@ -1,101 +1,133 @@ -## A gateway daemon for Kalatori +## A Gateway Daemon for Kalatori +!!! KALATORI IS IN PUBLIC BETA !!! + +Kalatori is an open-source daemon designed to enable secure and scalable blockchain payment processing. Licensed under GPLv3 ([LICENSE](LICENSE)), Kalatori currently supports assets on the Polkadot relay chain and its parachains. + +The daemon derives unique accounts for each payment using a provided seed phrase and outputs all payments to a specified recipient wallet. It also offers limited transaction tracking for order management. Kalatori operates in a multithreaded mode and supports multiple currencies configured in a simple TOML-based configuration file. + +Client facing frontends can communicate with Kalatori leveraging exposed API described in the [API documentation](https://alzymologist.github.io/kalatori-api). + +--- ### Download -Compiled binaries for Linux x86-64 can be found in the "Releases" section. +Download the latest Docker container or x86-64 release from the [GitHub releases page](https://github.com/Alzymologist/Kalatori-backend/releases/latest). -### Compile from the source +--- -To compile the daemon, the latest stable Rust compiler version is required. Then run the following command: +### Compile from Source + +To compile the daemon, ensure you have the latest stable version of the Rust compiler installed. Then, run: ```sh -cargo b -r --workspace +cargo build --release --workspace +``` +The compiled binaries will be located in the `target/release` path. + +### Project Structure + +- `chopsticks`: Contains configuration files for the Chopsticks tool and a Docker Compose setup for spawning Polkadot and AssetHub test chains. +- `configs`: Contains configuration files for supported chains and assets. +- `docs`: Includes project documentation. +- `src`: The source code for the Kalatori daemon. +- `tests`: Black-box test suite with a Docker Compose setup for testing the daemon. +- `Dockerfile`: Instructions for building a Docker image of the daemon. + +### Configuration File Example + +For Polkadot and Asset Hub chains, the configuration file should look like this: + +```toml +account-lifetime = 604800000 # 1 week. +debug = true +depth = 86400000 # 1 day. + +[[chain]] +name = "polkadot" +native-token = "DOT" +decimals = 10 +endpoints = [ + "wss://rpc.polkadot.io", + "wss://1rpc.io/dot", +] + +[[chain]] +name = "statemint" +endpoints = [ + "wss://polkadot-asset-hub-rpc.polkadot.io", + "wss://statemint-rpc.dwellir.com", +] + +[[chain.asset]] +name = "USDC" +id = 1337 + +[[chain.asset]] +name = "USDt" +id = 1984 ``` -Compiled binaries can be found in the `target/release` path. - -### Structure & settings -The daemon for Kalatori consists of 2 variants: -- `kalatori` may be used for DOT, the native currency of the Polkadot and Polkadot Asset Hub chains. -- `kalatori-ah` may be used for the Polkadot Asset Hub chain and 2 of its assets: USDt (1984) & USD Coin (1337). +### Environment variables -Both variants have almost the same startup environment variables: -- KALATORI_HOST: an address where the daemon opens its TCP socket server. -- KALATORI_SEED: a seed that's used as a base for the account derivation. -- KALATORI_DATABASE: a path to the daemon future/existing database. -> Note that a separate database file must be used for each supported currency, otherwise the database will be corrupted. -- KALATORI_RPC: an address of a Substrate RPC server. -- KALATORI_OVERRIDE_RPC: add this variable with any value to allow changing an RPC server address in the database. -- KALATORI_DECIMALS: set decimals for the chain native currency. -> Presents only in `kalatori`. -- KALATORI_USD_ASSET: sets which USD asset should be used. Possible value is "USDT" or "USDC". -> Presents only in `kalatori-ah`. -- KALATORI_DESTINATION: a hexadecimal address of the account that the daemon will send all payments to. +Kalatori requires the following environment variables for configuration: +- `KALATORI_HOST`: Address for the daemon's TCP socket server. +- `KALATORI_SEED`: Seed phrase for account derivation. +- `KALATORI_CONFIG`: Path to the chain configuration file in the configs directory. +- `KALATORI_RECIPIENT`: The hexadecimal address to which received payments will be transferred. +- `KALATORI_REMARK`: A string added to the transaction's remark field. -### Examples +### Usage Example -A tipical command to run `kalatori` for the Polkadot chain may look like this: +Run Kalatori for the Polkadot chain: ```sh KALATORI_HOST="127.0.0.1:16726" \ +KALATORI_CONFIG="configs/polkadot.toml" \ KALATORI_SEED="bottom drive obey lake curtain smoke basket hold race lonely fit walk" \ -KALATORI_DATABASE="database.redb" \ -KALATORI_RPC="wss://rpc.polkadot.io" \ -KALATORI_DECIMALS="12" \ -KALATORI_DESTINATION="0xd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d" \ +KALATORI_RECIPIENT="5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY" \ +KALATORI_REMARK="test" \ kalatori -``` +```` -And a command to run `kalatori-ah`for the Polkadot AssetHub chain may look like this: +### Testing -```sh -KALATORI_HOST="127.0.0.1:16726" \ -KALATORI_SEED="bottom drive obey lake curtain smoke basket hold race lonely fit walk" \ -KALATORI_DATABASE="database-ah-usdc.redb" \ -KALATORI_RPC="wss://polkadot-asset-hub-rpc.polkadot.io" \ -KALATORI_DESTINATION="0xd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d" \ -KALATORI_USD_ASSET="USDC" -kalatori-ah -``` +The black-box test suite verifies the daemon's functionality by interacting with a running instance. Use the following steps to set it up: +1. Start the daemon and test environment: + ```sh + cd tests + docker-compose up + ``` +2. Run the tests manually using Yarn: + ```sh + ct tests/kalatori-api-test-suite + yarn + yarn test + ``` -### Testing +Ensure the `DAEMON_HOST` environment variable points to the running daemon (default: `localhost:16726`). -[Chopsticks](https://github.com/AcalaNetwork/chopsticks) can be used to test the daemon out on a copy of a real network. This repository contains 2 config examples for testing: +For more details, refer to the [testing suite README.md](tests/kalatori-api-test-suite/README.md). -### - Polkadot +### Contributing -Use the following command inside this repository root directory to run Chopstick with the Polkadot config example: +We welcome contributions! Please refer to the [CONTRIBUTING.md](CONTRIBUTING.md) file for guidelines on contributing and submitting pull requests. -```sh -npx @acala-network/chopsticks@latest -c chopsticks/pd.yml -``` +### License -Then run `kalatori` with `KALATORI_RPC` set on the Chopsticks default server: +Kalatori is open-source software licensed under the GPLv3 License. See the [LICENSE](LICENSE) file for more details. -```sh -KALATORI_HOST="127.0.0.1:16726" \ -KALATORI_SEED="bottom drive obey lake curtain smoke basket hold race lonely fit walk" \ -KALATORI_RPC="ws://localhost:8000" \ -KALATORI_DECIMALS="12" \ -KALATORI_DESTINATION="0xd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d" \ -kalatori -``` +### Community and Support -### - Polkadot Asset Hub +Join the discussion and get support on: +- [Kalatori Matrix](https://matrix.to/#/#Kalatori-support:matrix.zymologia.fi) +- [GitHub Discussions](https://github.com/Alzymologist/Kalatori-backend/discussions) -Use the following command inside this repository root directory to run Chopstick with the Polkadot Asset Hub config example: +### Roadmap -```sh -npx @acala-network/chopsticks@latest -c chopsticks/pd-ah.yml -``` +Refer to the Kalatori project [board](https://github.com/orgs/Alzymologist/projects/2) and [milestones](https://github.com/Alzymologist/Kalatori-backend/milestones) for the current roadmap and upcoming features. -Then run `kalatori-ah` with `KALATORI_RPC` set on the Chopsticks default server, and `KALATORI_USD_ASSET` set on the USD asset being tested: +### Acknowledgments + +- Polkadot community +- Liberland team -```sh -KALATORI_HOST="127.0.0.1:16726" \ -KALATORI_SEED="bottom drive obey lake curtain smoke basket hold race lonely fit walk" \ -KALATORI_RPC="ws://localhost:8000" \ -KALATORI_DESTINATION="0xd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d" \ -KALATORI_USD_ASSET="USDC" -kalatori-ah -``` diff --git a/configs/chopsticks-lleo.toml b/configs/chopsticks-lleo.toml new file mode 100644 index 0000000..05b299b --- /dev/null +++ b/configs/chopsticks-lleo.toml @@ -0,0 +1,25 @@ +account-lifetime = 86400000 # 1 day. +depth = 3600000 # 1 hour. +debug = true + +[[chain]] +name = "rococo" +native-token = "DOT" +decimals = 10 +endpoints = [ + "wss://node-polkadot.zymologia.fi", +] + +[[chain]] +name = "statemint" +endpoints = [ + "wss://node-polkadot-ah.zymologia.fi", +] + +[[chain.asset]] +name = "USDC" +id = 1337 + +[[chain.asset]] +name = "USDt" +id = 1984 diff --git a/docs/DOCS.md b/docs/DOCS.md deleted file mode 100644 index 3e7ee3c..0000000 --- a/docs/DOCS.md +++ /dev/null @@ -1,82 +0,0 @@ -# Kalatori docs - - -### Decisions -1. Amounts in tx/rx is f64, daemon stores it as u128 - -f64 amount is rounded up in frontend; down in backend - to make sure order always passes - -Reasoning: - -- JSON messes up long integers routinely (no exact implementation is defined) -- nobody cares about significant figures outside of u64 precision -- this supports a good natured philosophy of friendly haggling and discounts that's natural for good business - -# Daemon API - -## Overview - -API for interacting with the daemon. - -## Endpoints - -### Health Check - -- **Endpoint**: `GET /health` -- **Description**: Returns the status of the API and its current version. - -- **Response (200 OK)**: - ```json - { - "version": string - "chain": string, - "code": string, - } - ``` - -### Create or get Account if it already exists - -- **Endpoint**: `POST /account` -- **Description**: Creates a new account, returns account if exists already - -- **Request**: - ```json - { - "order_id": string, - "amount": float, - } - ``` - -- **Response (201 Created/200 OK)**: - ```json - { - "status": string:pending|paid, - "address": string, - "order_id": string, - "rpc": string, - "decimals": integer, - "amount": float - } - ``` - -- **Response(400 Bad Request)**: - ```json - { - "error": string - } - ``` - - -## Common Responses for Errors - -- **404 Not Found**: - - Empty body - -- **500 Internal Server Error**: - ```json - { - "error": string - } - ``` - - diff --git a/docs/structure.md b/docs/structure.md deleted file mode 100644 index 89ef24a..0000000 --- a/docs/structure.md +++ /dev/null @@ -1,176 +0,0 @@ -The daemon consists of 5 modules: - -* `callback.rs` -* `database.rs` -* `main.rs` -* `rpc.rs` -* `server.rs` - -## `main.rs` - -Everything starts here. Previously, the start logic was in `lib.rs` because this allowed to add documentation directly to the source code (e.g., for the server API), but since Markdown isn't sufficient for complex descriptions & generating the documentation with `rustdoc` just to read it is inconvenient for those who won't build the daemon from the source, it was decided to use only `main.rs` while keeping the documentation elsewhere. - -At the start, the daemon reads the following environment variables: - -* `KALATORI_CONFIG` - -The path to a config file. - -* `KALATORI_LOG` - -[Filter directives for the logger.](https://docs.rs/tracing-subscriber/0.3/tracing_subscriber/filter/struct.EnvFilter.html) - -* `KALATORI_SEED` - -The seed phrase that new accounts will be derived from. - -* `KALATORI_OLD_SEED_*` - -Old seed phrases for existing accounts. Used for the unimplemented seed rotation logic. - -* `KALATORI_RECIPIENT` - -The recipient account address in the SS58 format. - -* `KALATORI_REMARK` - -The arbitrary string to be passed with the server info from the server API. - -Then the daemon parses a config file. The config format can be found in at the end of `main.rs`. Examples are in the [configs](../configs) directory. - -## `server.rs` - -Contains the server & server API implementation using the `axum` framework. Currently, everything is hardcoded to USDC from the Polkadot Asset Hub parachain. - -The amount parameter in API is treated as `f64` during a de- and serialization due to restrictions on the frontend side. This approach is error-prone because `f64` is insufficient to hold `u128` (that's the actual type of the amount parameter) leading to the loss of last digits during a parsing/formatting. Moreover, the converting between `f64` & `u128` suffers from rounding errors that could unexpectedly mutate a lenghty number during a roundtrip. An example of such a number can be found in the test at the end of `main.rs`. A proposed solution to this problem was to create a custom (de)serializer that'd parse/format floats in the amount parameter directly to/from `u128` skipping the erroneous parts with `f64`, but as it's a quite long task, it was decided to stick with the default (de)serializer with `f64` and hope to get no lenghty amounts. - -## `callback.rs` - -Contains a simple function to make callbacks with the order info. To do this, it was decided to use the `ureq` crate, a blocking HTTP-client. To prevent blocking the async executor, every callback is spawned in the Tokio's blocking threadpool. Since we don't need any complex HTTP 1 features as well as HTTP 2/3, `ureq` is a good choice for a rather dependencies-heavy crate that the daemon is. - -## `rps.rs` - -The most complex module of the daemon where all the crucial blockchain-related logic happens. In the first version of daemon, only one chain & currency (a native token or an asset) were supported in 2 separate binaries. With the requirement to support both a native token & multiple assets within a single daemon, a new architecture was needed. Since from Substrate RPC's POV there's no difference between relaychains & parachains, it was decided to make the support of any number of chains with the parallel processing, different configurations & currencies, not just a pair of a relaychain with a native token & a parachain with assets in 2 threads. The config file was introduced to process chain configurations from a much more convenient format than a cluster of environment variables. For now, the daemon can parse this config and prepare chains & their currencies for the block parsing loop. The block parsing loop isn't implemented. - -### Assumptions - -The current Substrate RPC API & pallets used by the daemon, unfortunately, don't have any strict standard, which all nodes & chains follow, and the common implementations of RPC API & pallets lack of some essential features. This creates restrictions & assumptions for which nodes & chains the daemon can be used with. Of course, our first priority is and will be the support of the Polkadot relaychain & the Polkadot Asset Hub parachain with the support of USDT & USDC assets on that parachain, but in the future, we'll try to lift the following limitations and support a broader range of nodes & chains along with the evolution of the Substrate ecosystem & its standards. - -#### [`ChargeAssetTxPayment`] - -One of difficult parts in the chain preparation is to determine & dynamically process the asset type in [`ChargeAssetTxPayment`], the signed extension that's used to pay transaction fees in an asset instead of a native token. Kusama, Westend & Rococo Asset Hub parachains have the [`MultiLocation`] struct as the asset type, and only Polkadot Asset Hub has the `u32` type as its asset type. There's also a possibility of other types, but the daemon supports only these 2 because they're the most common. The daemon chooses between them by looking up the properties of [`ChargeAssetTxPayment`] in the metadata returned from an RPC server. To avoid depending on the `staging-xcm` crate, the daemon in `asset.rs` has the copy of [`MultiLocation`] from this crate with custom trait implementations to process it along with `u32`. - -#### Transfers - -All extrinsics sent by the daemon are transfer transactions. Despite they themselves are quite unsophisticated, the whole process of *the approximate fee calculation to subtract it from the transfer amount and then, in case of a fail because a real fee was higher than the daemon estimated, the resending of the same transaction after another never accurate calculation* does sound like a really hard path. - -We tried to use the [`transfer_allow_death`] call in the Balances pallet for all transfers because we thought that a transfer fee would never be greater than the existential deposit but that isn't true for all chains (e.g., it's true for Polkadot but not for Kusama), and unlikely will be a standard, so we've stuck with the [`transfer_all`] call for transfers to both the beneficiary & overpayers accounts. - -The Assets pallet doesn't have a call similar to Balances's [`transfer_all`], so the daemon uses [`transfer`] assuming that a fee for that call wouldn't be higher than asset's [`min_balance`]. For now, that's true for USDT & USDC on Polkadot Asset Hub, but **if one these or another assets won't meet this criteria, the daemon won't be able to work with them**. We plan to propose and hopefully include some kind of the `transfer_all` call in Assets pallet to subsequently eliminate this restriction. - -#### Transactions - -The daemon uses the `author_submit_extrinsic` method to submit transactions. Another option to do this is the `author_submit_and_watch_extrinsic` method, it was considered more reliable, albeit more heavy & restricted, because it provides a subscription to a status of every submitted transaction, but its main problem is there's no way to resume a subscription (e.g., in case of an abnormal daemon shutdown or a loss of a connection with an RPC server). To track a transactions status, the daemon will use the [CheckMortality](https://docs.rs/frame-system/32/frame_system/struct.CheckMortality.html) signed extension. It's used to add mortality to a transaction, mortality measured with blocks, so the daemon can calculate a block number on which a transaction would be dead (invalid) and wait for it. If the transaction hasn't appeared before its death block, the transaction considered lost. It might happen for a variety of reasons that the daemon doesn't have the control of, e.g., a connection loss, faulty RPC server, unusual chain's runtime implementation. - -Unresolved questions: - -* What should the daemon do in the case of a lost transaction? - -One of expected behaviours would be to send it again, but it's unclear how many attempts should be made before giving up. - -* How should the transaction mortality affect account lifetimes in the database? (See the description of the `"hitlist"` database table.) - -## `database.rs` - -To store data, the daemon uses the `redb` crate, a key-value store with tables. The daemon database consists of the following tables: - -* `"root"` -* `"keys"` -* `"chains"` -* `"invoices"` -* `"accounts*"` -* `"transactions*"` -* `"hit_list*"` - -### `"root"` - -Contains the database format version & the daemon info. The daemon info tracks the information about chains & keys the daemon is using. - -#### Chains - -Chains are stored as `Map`. `String` is a arbitrary name of a chain from the daemon config. Chain properties consists of a chain's hash, genesis hash, kind, and assets. Chain hash is used as an internal key to a chain in other tables instead of a string from the config to enable the renaming of a chain without changing its internal key. A genesis hash is used to check whether given RPC endpoints represent a right chain. A kind means whether a chain operates with [`MultiLocation`] or `u32` as the asset type (if a chain doesn't have assets, `u32` is the default). - -#### Keys - -Accounts for new invoices are derived from the current key. If the database has no invoices, the current key can be replaced with another one from the `KALATORI_SEED` environment variable. If the database has invoices, the current key can be replaced in the same way, but the previous current key's seed must be moved to a new `KALATORI_OLD_SEED_*` variable. Then the daemon will match old seeds with public keys stored in the database. This is used for the key rotation logic. - -### `"keys"` - -The contuation of the above paragraph. Keys are also stored here to track a number of account created with a certain key. Once the number reaches 0, its key is deleted from the database, and the daemon won't require its seed in `KALATORI_OLD_SEED_*` variables. - -This data is stored in separated database because the `"root"` table stores data in encoded form requiring a full decode & encode roundtrip for each slot change. - -### `"chains"` - -Used to store the last saved block for each chain. - -### `"invoices"` - -Invoices are stored as `Map`. The key has an arbitrary format, and set by the server module that in turn receives it from the frontend. - -An invoice contains: - -* The public key & the hard derivation joint. - -The payment account address can be generated from them. - -In the first version of the daemon, the database didn't store the invoice's derivation joint and calculated it from the invoice's string key received from the server module. There was a very small chance of the hash collision that could lead to, e.g., returning information about already paid invoice instead of creating a new one, so it was decided to use a calculated derivation joint as the initial hash, check if the database contains an account from the joint, and if not, create an account from the initial joint, or calculate an unoccupied joint and create an account with it. - -* The payment status. - -It's unclear how to properly separate unpaid accounts for the unpaid account checking on the daemon startup. There are a few possible options: iterate over all invoices filtering unpaid ones out; storing paid & unpaid invoices in separate tables; storing only unpaid invoices' keys in a separate table. - -* The creation timestamp. - -* The price. - -Must be checked to be greater than its currency's existential deposit. - -* The callback. - -Used to send invoice statuses back to the frontend. Ignored if empty. - -* The message. - -A message with an arbitrary human-readable format. Usually contains error messages. - -* Transactions. - -Finalized & pending transfer transactions. Each transaction contains the recipient, the amount, the string with the hex-encoded transaction, and the currency info (since each invoice has only 1 currency, it's unclear why the server API needs the currency info for every invoice's transaction). - -> The next tables have `*` at the end of their name. It's the placeholder for a chain hash. The daemon creates a set of these table for each chain. - -### `"accounts*"` - -Stores account addresses with currency info (`Map<(Option, AccountId), InvoiceKey>`) for the block processing loop. What's notably 1 account address can be used for different currencies in case of an aforementioned hash collision because while invoices are created for 1 currency, their account's transactions don't depend on currencies, so the daemon should be able to process multi-currency accounts without conflicts. - -### `"transactions*"` - -Stores info about pending transfer transactions with their death block as the key. - -### `"hit_list*"` - -The content of this table is in progress. It should be used to track lifetimes of invoices to remove them from the database, thereby maintaining the stable element search time and avoiding the unused accounts tracking for overpayments. Currently, this table looks like `Map, Account)>`, but this layout is insufficient because of the following - -Unresolved questions: - -* On a startup, the daemon simply removes all dead invoices. But consider the following scenario: the daemon shuts down (due to a crash or manually) after creating an invoice, invoice's account receives money, and after a time longer than the invoice's lifetime, the daemon starts up and immediately deletes the dead but paid invoice. How to avoid this situation? Should the daemon check on startup all unpaid invoices, and renew lifetimes of just paid ones? What about overpayments on paid invoices? Should the daemon renew lifetimes of all invoices then? - -* How transaction lifetimes should affect ones of invoices? Currently, if some transaction appears in a block after its invoice's lifetime, the transaction is ignored because its invoice have been deleted. This applies to both incoming & outgoing refund/withdrawal transactions. Lifetimes of transactions aren't fixed values, their upper bound depends on the chains' `BlockHashCount` runtime parameter that can be different on each chain and modified on a runtime upgrade, and the lower bound depends on a block producing/finalization algorithm and properties of a connection with an RPC server. - -[`MultiLocation`]: https://docs.rs/staging-xcm/11/staging_xcm/v3/struct.MultiLocation.html -[`transfer_all`]: https://docs.rs/pallet-balances/33/pallet_balances/pallet/struct.Pallet.html#method.transfer_all -[`transfer_allow_death`]: https://docs.rs/pallet-balances/33/pallet_balances/pallet/struct.Pallet.html#method.transfer_allow_death -[`transfer`]: https://docs.rs/pallet-assets/33/pallet_assets/pallet/struct.Pallet.html#method.transfer -[`min_balance`]: https://docs.rs/pallet-assets/33/src/pallet_assets/types.rs.html#66 -[`ChargeAssetTxPayment`]: https://docs.rs/pallet-asset-tx-payment/32/pallet_asset_tx_payment/struct.ChargeAssetTxPayment.html diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index 7282c9a..d829f8a 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -11,6 +11,8 @@ services: volumes: - ../chopsticks/pd.yml:/app/config.yml command: ["chopsticks", "-c", "/app/config.yml", "-p", "8000", "--addr", "0.0.0.0"] + networks: + - kalatori-network chopsticks-statemint: build: @@ -22,6 +24,8 @@ services: volumes: - ../chopsticks/pd-ah.yml:/app/config.yml command: ["chopsticks", "-c", "/app/config.yml", "-p", "9000", "--addr", "0.0.0.0"] + networks: + - kalatori-network kalatori-rust-app: build: @@ -42,6 +46,9 @@ services: - KALATORI_RECIPIENT=5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY - KALATORI_REMARK=test command: /bin/sh -c "sleep 10 && /app/kalatori" # 10-second sleep to ensure chopsticks is ready + networks: + - kalatori-network + tests: image: node:20 @@ -53,3 +60,9 @@ services: environment: - DAEMON_HOST=http://kalatori-daemon:16726 command: /bin/sh -c "sleep 180 && yarn install && yarn test" + networks: + - kalatori-network + +networks: + kalatori-network: + external: true