From b2a850a9a38b21cf4212afeb44d0a8dac6e4b5b1 Mon Sep 17 00:00:00 2001 From: John Adler Date: Tue, 27 Sep 2022 09:46:41 -0400 Subject: [PATCH 01/22] Initial specs migration. (#793) Fixes #792 An initial migration of the existing specs as of [`e59efd63a2165866584833e91e1cb8a6ed8c8203`](https://github.com/celestiaorg/celestia-specs/tree/e59efd63a2165866584833e91e1cb8a6ed8c8203). It's expected that there are a few broken links and extra content that has to be pruned, along with incorrect content. The important thing is to not miss anything. Also sets up a CI workflow that will build the book. --- .github/workflows/gh-pages.yml | 21 + specs/.gitignore | 1 + specs/README.md | 22 + specs/book.toml | 12 + specs/src/README.md | 9 + specs/src/SUMMARY.md | 11 + specs/src/rationale/index.md | 3 + specs/src/rationale/message_block_layout.md | 51 + specs/src/specs/block_proposer.md | 26 + specs/src/specs/consensus.md | 770 ++++++++++ specs/src/specs/data_structures.md | 994 ++++++++++++ .../specs/figures/block_data_structures.dot | 36 + .../specs/figures/block_data_structures.svg | 117 ++ specs/src/specs/figures/data_root.svg | 39 + specs/src/specs/figures/rs2d.svg | 102 ++ specs/src/specs/figures/rs2d_extend.svg | 106 ++ specs/src/specs/figures/rs2d_extending.svg | 120 ++ .../figures/rs2d_originaldata_message.svg | 34 + .../figures/rs2d_originaldata_reserved.svg | 30 + specs/src/specs/figures/rs2d_quadrants.svg | 112 ++ specs/src/specs/figures/rs2d_row.svg | 100 ++ specs/src/specs/figures/share.dot | 25 + specs/src/specs/figures/share.svg | 36 + specs/src/specs/index.md | 6 + specs/src/specs/networking.md | 122 ++ specs/src/specs/proto/consensus.proto | 47 + specs/src/specs/proto/types.proto | 285 ++++ specs/src/specs/proto/wire.proto | 98 ++ specs/theme/highlight.js | 1328 +++++++++++++++++ 29 files changed, 4663 insertions(+) create mode 100644 .github/workflows/gh-pages.yml create mode 100644 specs/.gitignore create mode 100644 specs/README.md create mode 100644 specs/book.toml create mode 100644 specs/src/README.md create mode 100644 specs/src/SUMMARY.md create mode 100644 specs/src/rationale/index.md create mode 100644 specs/src/rationale/message_block_layout.md create mode 100644 specs/src/specs/block_proposer.md create mode 100644 specs/src/specs/consensus.md create mode 100644 specs/src/specs/data_structures.md create mode 100644 specs/src/specs/figures/block_data_structures.dot create mode 100644 specs/src/specs/figures/block_data_structures.svg create mode 100644 specs/src/specs/figures/data_root.svg create mode 100644 specs/src/specs/figures/rs2d.svg create mode 100644 specs/src/specs/figures/rs2d_extend.svg create mode 100644 specs/src/specs/figures/rs2d_extending.svg create mode 100644 specs/src/specs/figures/rs2d_originaldata_message.svg create mode 100644 specs/src/specs/figures/rs2d_originaldata_reserved.svg create mode 100644 specs/src/specs/figures/rs2d_quadrants.svg create mode 100644 specs/src/specs/figures/rs2d_row.svg create mode 100644 specs/src/specs/figures/share.dot create mode 100644 specs/src/specs/figures/share.svg create mode 100644 specs/src/specs/index.md create mode 100644 specs/src/specs/networking.md create mode 100644 specs/src/specs/proto/consensus.proto create mode 100644 specs/src/specs/proto/types.proto create mode 100644 specs/src/specs/proto/wire.proto create mode 100644 specs/theme/highlight.js diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml new file mode 100644 index 0000000000..157c1c3077 --- /dev/null +++ b/.github/workflows/gh-pages.yml @@ -0,0 +1,21 @@ +name: github pages + +on: + push: + branches: + - main + pull_request: + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Setup mdBook + uses: peaceiris/actions-mdbook@v1 + with: + mdbook-version: "0.4.21" + + - name: Build book + run: mdbook build specs diff --git a/specs/.gitignore b/specs/.gitignore new file mode 100644 index 0000000000..7585238efe --- /dev/null +++ b/specs/.gitignore @@ -0,0 +1 @@ +book diff --git a/specs/README.md b/specs/README.md new file mode 100644 index 0000000000..d9ef8443d2 --- /dev/null +++ b/specs/README.md @@ -0,0 +1,22 @@ +# Celestia App Specifications + +## Building From Source + +To build book: + +```sh +mdbook build +``` + +To serve locally: + +```sh +mdbook serve +``` + +## Contributing + +Markdown files must conform to [GitHub Flavored Markdown](https://github.github.com/gfm/). Markdown must be formatted with: + +- [markdownlint](https://github.com/DavidAnson/markdownlint) +- [Markdown Table Prettifier](https://github.com/darkriszty/MarkdownTablePrettify-VSCodeExt) diff --git a/specs/book.toml b/specs/book.toml new file mode 100644 index 0000000000..694ba1019b --- /dev/null +++ b/specs/book.toml @@ -0,0 +1,12 @@ +[book] +authors = ["Celestia Labs"] +language = "en" +multilingual = false +src = "src" +title = "Celestia App Specifications" + +[output.html] +git-repository-url = "https://github.com/celestiaorg/celestia-app" + +[rust] +edition = "2021" diff --git a/specs/src/README.md b/specs/src/README.md new file mode 100644 index 0000000000..445ac7f3c0 --- /dev/null +++ b/specs/src/README.md @@ -0,0 +1,9 @@ +# Celestia App Specifications + +- [Specification](./specs/index.md) + - [Data Structures](./specs/data_structures.md) + - [Consensus](./specs/consensus.md) + - [Block Proposer](./specs/block_proposer.md) + - [Networking](./specs/networking.md) +- [Rationale](./rationale/index.md) + - [Message Layout](./rationale/message_block_layout.md) diff --git a/specs/src/SUMMARY.md b/specs/src/SUMMARY.md new file mode 100644 index 0000000000..af7f74b048 --- /dev/null +++ b/specs/src/SUMMARY.md @@ -0,0 +1,11 @@ +# Summary + +[Celestia App Specifications](./README.md) + +- [Specification](./specs/index.md) + - [Data Structures](./specs/data_structures.md) + - [Consensus](./specs/consensus.md) + - [Block Proposer](./specs/block_proposer.md) + - [Networking](./specs/networking.md) +- [Rationale](./rationale/index.md) + - [Message Layout](./rationale/message_block_layout.md) diff --git a/specs/src/rationale/index.md b/specs/src/rationale/index.md new file mode 100644 index 0000000000..f6459b7015 --- /dev/null +++ b/specs/src/rationale/index.md @@ -0,0 +1,3 @@ +# Rationale + +- [Message Layout](./message_block_layout.md) diff --git a/specs/src/rationale/message_block_layout.md b/specs/src/rationale/message_block_layout.md new file mode 100644 index 0000000000..e8b7757901 --- /dev/null +++ b/specs/src/rationale/message_block_layout.md @@ -0,0 +1,51 @@ +# Message Layout + +- [Preamble](#preamble) +- [Message Layout Rationale](#message-layout-rationale) + - [Non-Interactive Default Rules](#non-interactive-default-rules) + - [Caveats](#caveats) + +## Preamble + +Celestia uses [a data availability scheme](https://arxiv.org/abs/1809.09044) that allows nodes to determine whether a block's data was published without downloading the whole block. The core of this scheme is arranging data in a two-dimensional matrix then applying erasure coding to each row and column. This document describes the rationale for how data—transactions, messages, and other data—[is actually arranged](../specs/data_structures.md#arranging-available-data-into-shares). Familiarity with the [originally proposed data layout format](https://arxiv.org/abs/1809.09044) is assumed. + +## Message Layout Rationale + +Block data consists of transactions (which modify the Celestia chain's state), intermediate state roots (required for fraud proofs of the aforementioned transactions), messages (binary blobs which do not modify the Celestia state, but which are intended for a Celestia application identified with a provided namespace ID), and other relevant pieces of data (e.g. evidence for slashing). We want to arrange this data into a `k * k` matrix of fixed-sized shares, which will later be committed to in [Namespace Merkle Trees (NMTs)](../specs/data_structures.md#namespace-merkle-tree). + +The simplest way we can imagine arranging block data is to simply serialize it all in no particular order, split it into fixed-sized shares, then arrange those shares into the `k * k` matrix in row-major order. However, this naive scheme can be improved in a number of ways, described below. + +First, we impose some ground rules: + +1. Data must be ordered by namespace ID. This makes queries into a NMT commitment of that data more efficient. +1. Since non-message data are not naturally intended for particular namespaces, we assign reserved namespaces for them. A range of namespaces is reserved for this purpose, starting from the lowest possible namespace ID. +1. By construction, the above two rules mean that non-message data always precedes message data in the row-major matrix, even when considering single rows or columns. +1. Data with different namespaces must not be in the same share. This might cause a small amount of wasted block space, but makes the NMT easier to reason about in general since leaves are guaranteed to belong to a single namespace. + +Transactions can pay fees for a message to be included in the same block as the transaction itself. However, we do not want serialized transactions to include the entire message they pay for (which is the case in other blockchains with native execution, e.g. calldata in Ethereum transactions or OP_RETURN data in Bitcoin transactions), otherwise every node that validates the sanctity of the Celestia coin would need to download all message data. Transactions must therefore only include a commitment to (i.e. some hash of) the message they pay fees for. If implemented naively (e.g. with a simple hash of the message, or a simple binary Merkle tree root of the message), this can lead to a data availability problem, as there are no guarantees that the data behind these commitments is actually part of the block data. + +To that end, we impose some additional rules onto _messages only_: messages must be placed is a way such that both the transaction sender and the block producer can be held accountable—a necessary property for e.g. fee burning. Accountable in this context means that + +1. The transaction sender must pay sufficient fees for message inclusion. +1. The block proposer cannot claim that a message was included when it was not (which implies that a transaction and the message it pays for must be included in the same block). + +Specifically, messages must begin at a new share, unlike non-message data which can span multiple shares. We note a nice property from this rule: if the transaction sender knows 1) `k`, the size of the matrix, 2) the starting location of their message in a row, and 3) the length of the message (they know this since they are sending the message), then they can actually compute a sequence of roots to _subtrees in the row NMTs_. More importantly, anyone can compute this, and can compute _the simple Merkle root of these subtree roots_. + +This, however, requires the block producer to interact with the transaction sender to provide them the starting location of their message. This can be done selectively, but is not ideal as a default for e.g. end-user wallets. + +### Non-Interactive Default Rules + +As a non-consensus-critical default, we can impose some additional rules on message placement to make the possible starting locations of messages sufficiently predictable and constrained such that users can deterministically compute subtree roots without interaction: + +1. Messages that span multiple rows must begin at the start of a row (this can occur if a message is longer than `k` shares _or_ if the block producer decides to start a message partway through a row and it cannot fit). +1. Messages begin at a location aligned with the largest power of 2 that is not larger than the message length or `k`. + +With the above constraints, we can compute subtree roots deterministically easily: simply slice off either the largest power of 2 that isn't larger than the remaining message length, or `k`, whichever is smaller. This is done recursively. As an example, with `k = 4` and message length of `11`, the message would be sliced with lengths `4, 4, 2, 1`. The resulting slices are the leaves of subtrees whose roots can be computed. Due to the rules above, the subtrees are guaranteed to be aligned to powers of 2, and thus the subtree roots will be present as internal nodes in the NMT of _some_ row(s). + +This is similar to [Merkle Mountain Ranges](https://www.usenix.org/legacy/event/sec09/tech/full_papers/crosby.pdf), though with the largest subtree bounded by `k` rather than being unbounded. + +The last piece of the puzzle is determining _which_ row the message is placed at (or, more specifically, the starting location). This is needed to keep the block producer accountable. To this end, the block producer simply augments each fee-paying transaction with some metadata: the starting location of the message the transaction pays for. + +### Caveats + +The message placement rules described above conflict with the first rule that shares must be ordered by namespace ID, as shares between two messages that are not placed adjacent to each other do not have a natural namespace they belong to. This is resolved by requiring that such shares have a value of zero and a namespace ID equal to the preceding message's. Since their value is known, they can be omitted from NMT proofs of all shares of a given namespace ID. diff --git a/specs/src/specs/block_proposer.md b/specs/src/specs/block_proposer.md new file mode 100644 index 0000000000..948a5ecb78 --- /dev/null +++ b/specs/src/specs/block_proposer.md @@ -0,0 +1,26 @@ +# Honest Block Proposer + +- [Deciding on a Block Size](#deciding-on-a-block-size) +- [Laying out Transactions and Messages](#laying-out-transactions-and-messages) + +This document describes the tasks of an honest block proposer to assemble a new block. Performing these actions is not enforced by the [consensus rules](./consensus.md), so long as a valid block is produced. + +## Deciding on a Block Size + +Before [arranging available data into shares](./data_structures.md#arranging-available-data-into-shares), the size of the original data's square must be determined. + +There are two restrictions on the original data's square size: + +1. It must be at most [`AVAILABLE_DATA_ORIGINAL_SQUARE_MAX`](./consensus.md#constants). +1. It must be a power of 2. + +With these restrictions in mind, the block proposer performs the following actions: + +1. Collect as many transactions and messages from the mempool as possible, such that the total number of shares is at most [`AVAILABLE_DATA_ORIGINAL_SQUARE_MAX`](./consensus.md#constants). +1. Compute the smallest square size that is a power of 2 that can fit the number of shares. +1. Attempt to [lay out the collected transactions and messages](#laying-out-transactions-and-messages) in the current square. + 1. If the square is too small to fit all transactions and messages (which may happen [due to needing to insert padding between messages](../rationale/message_block_layout.md)) and the square size is smaller than [`AVAILABLE_DATA_ORIGINAL_SQUARE_MAX`](./consensus.md#constants), double the size of the square and repeat the above step. + +Note: the maximum padding shares between messages should be at most twice the number of message shares. Doubling the square size (i.e. quadrupling the number of shares in the square) should thus only have to happen at most once. + +## Laying out Transactions and Messages diff --git a/specs/src/specs/consensus.md b/specs/src/specs/consensus.md new file mode 100644 index 0000000000..71fdc7e5f2 --- /dev/null +++ b/specs/src/specs/consensus.md @@ -0,0 +1,770 @@ +# Consensus Rules + +- [System Parameters](#system-parameters) + - [Units](#units) + - [Constants](#constants) + - [Reserved Namespace IDs](#reserved-namespace-ids) + - [Reserved State Subtree IDs](#reserved-state-subtree-ids) + - [Rewards and Penalties](#rewards-and-penalties) +- [Leader Selection](#leader-selection) +- [Fork Choice](#fork-choice) +- [Block Validity](#block-validity) +- [Block Structure](#block-structure) + - [`block.header`](#blockheader) + - [`block.availableDataHeader`](#blockavailabledataheader) + - [`block.lastCommit`](#blocklastcommit) + - [`block.availableData`](#blockavailabledata) +- [State Transitions](#state-transitions) + - [`block.availableData.evidenceData`](#blockavailabledataevidencedata) + - [`block.availableData.transactionData`](#blockavailabledatatransactiondata) + - [SignedTransactionDataTransfer](#signedtransactiondatatransfer) + - [SignedTransactionDataMsgPayForData](#signedtransactiondatamsgpayfordata) + - [SignedTransactionDataCreateValidator](#signedtransactiondatacreatevalidator) + - [SignedTransactionDataBeginUnbondingValidator](#signedtransactiondatabeginunbondingvalidator) + - [SignedTransactionDataUnbondValidator](#signedtransactiondataunbondvalidator) + - [SignedTransactionDataCreateDelegation](#signedtransactiondatacreatedelegation) + - [SignedTransactionDataBeginUnbondingDelegation](#signedtransactiondatabeginunbondingdelegation) + - [SignedTransactionDataUnbondDelegation](#signedtransactiondataunbonddelegation) + - [SignedTransactionDataBurn](#signedtransactiondataburn) + - [SignedTransactionRedelegateCommission](#signedtransactionredelegatecommission) + - [SignedTransactionRedelegateReward](#signedtransactionredelegatereward) + - [Begin Block](#begin-block) + - [End Block](#end-block) + +## System Parameters + +### Units + +| name | SI | value | description | +|------|-------|---------|---------------------| +| `1u` | `1u` | `10**0` | `1` unit. | +| `2u` | `k1u` | `10**3` | `1000` units. | +| `3u` | `M1u` | `10**6` | `1000000` units. | +| `4u` | `G1u` | `10**9` | `1000000000` units. | + +### Constants + +| name | type | value | unit | description | +|-----------------------------------------|----------|--------------|---------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `AVAILABLE_DATA_ORIGINAL_SQUARE_MAX` | `uint64` | | `share` | Maximum number of rows/columns of the original data [shares](data_structures.md#share) in [square layout](data_structures.md#arranging-available-data-into-shares). | +| `AVAILABLE_DATA_ORIGINAL_SQUARE_TARGET` | `uint64` | | `share` | Target number of rows/columns of the original data [shares](data_structures.md#share) in [square layout](data_structures.md#arranging-available-data-into-shares). | +| `BLOCK_TIME` | `uint64` | | second | Block time, in seconds. | +| `CHAIN_ID` | `string` | `"Celestia"` | | Chain ID. Each chain assigns itself a (unique) ID. | +| `GENESIS_COIN_COUNT` | `uint64` | `10**8` | `4u` | `(= 100000000)` Number of coins at genesis. | +| `MAX_GRAFFITI_BYTES` | `uint64` | `32` | `byte` | Maximum size of transaction graffiti, in bytes. | +| `MAX_VALIDATORS` | `uint16` | `64` | | Maximum number of active validators. | +| `NAMESPACE_ID_BYTES` | `uint64` | `8` | `byte` | Size of namespace ID, in bytes. | +| `NAMESPACE_ID_MAX_RESERVED` | `uint64` | `255` | | Value of maximum reserved namespace ID (inclusive). 1 byte worth of IDs. | +| `SHARE_RESERVED_BYTES` | `uint64` | `1` | `byte` | Bytes reserved at the beginning of each [share](data_structures.md#share). Must be sufficient to represent `SHARE_SIZE`. | +| `SHARE_SIZE` | `uint64` | `256` | `byte` | Size of transaction and message [shares](data_structures.md#share), in bytes. | +| `STATE_SUBTREE_RESERVED_BYTES` | `uint64` | `1` | `byte` | Number of bytes reserved to identify state subtrees. | +| `UNBONDING_DURATION` | `uint32` | | `block` | Duration, in blocks, for unbonding a validator or delegation. | +| `VERSION_APP` | `uint64` | `1` | | Version of the Celestia application. Breaking changes (hard forks) must update this parameter. | +| `VERSION_BLOCK` | `uint64` | `1` | | Version of the Celestia chain. Breaking changes (hard forks) must update this parameter. | + +### Reserved Namespace IDs + +| name | type | value | description | +|-----------------------------------------|---------------|----------------------|--------------------------------------------------------------------------------------------| +| `TRANSACTION_NAMESPACE_ID` | `NamespaceID` | `0x0000000000000001` | Transactions: requests that modify the state. | +| `INTERMEDIATE_STATE_ROOT_NAMESPACE_ID` | `NamespaceID` | `0x0000000000000002` | Intermediate state roots, committed after every transaction. | +| `EVIDENCE_NAMESPACE_ID` | `NamespaceID` | `0x0000000000000003` | Evidence: fraud proofs or other proof of slashable action. | +| `TAIL_TRANSACTION_PADDING_NAMESPACE_ID` | `NamespaceID` | `0x00000000000000FF` | Tail padding for transactions: padding after all transactions but before messages. | +| `TAIL_PADDING_NAMESPACE_ID` | `NamespaceID` | `0xFFFFFFFFFFFFFFFE` | Tail padding for messages: padding after all messages to fill up the original data square. | +| `PARITY_SHARE_NAMESPACE_ID` | `NamespaceID` | `0xFFFFFFFFFFFFFFFF` | Parity shares: extended shares in the available data matrix. | + +### Reserved State Subtree IDs + +| name | type | value | +|----------------------------------|------------------|--------| +| `ACCOUNTS_SUBTREE_ID` | `StateSubtreeID` | `0x01` | +| `ACTIVE_VALIDATORS_SUBTREE_ID` | `StateSubtreeID` | `0x02` | +| `INACTIVE_VALIDATORS_SUBTREE_ID` | `StateSubtreeID` | `0x03` | +| `DELEGATIONS_SUBTREE_ID` | `StateSubtreeID` | `0x04` | +| `MESSAGE_PAID_SUBTREE_ID` | `StateSubtreeID` | `0x05` | + +### Rewards and Penalties + +| name | type | value | unit | description | +|--------------------------|----------|-------------|--------|---------------------------------------------------------| +| `SECONDS_PER_YEAR` | `uint64` | `31536000` | second | Seconds per year. Omit leap seconds. | +| `TARGET_ANNUAL_ISSUANCE` | `uint64` | `2 * 10**6` | `4u` | `(= 2000000)` Target number of coins to issue per year. | + +## Leader Selection + +TODO + +## Fork Choice + +The Tendermint consensus protocol is fork-free by construction under an honest majority of stake assumption. + +If a block has a [valid commit](#blocklastcommit), it is part of the canonical chain. If equivocation [evidence](./data_structures.md#evidence) is detected for more than 1/3 of voting power, the node must halt. See [rationale doc](../rationale/fork_choice_das.md) for more information. + +## Block Validity + +The validity of a newly-seen block, `block`, is determined by two components, detailed in subsequent sections: + +1. [Block structure](#block-structure): whether the block header is valid, and data in a block is arranged into a valid and matching data root (i.e. syntax). +1. [State transition](#state-transitions): whether the application of transactions in the block produces a matching and valid state root (i.e. semantics). + +Pseudocode in this section is not in any specific language and should be interpreted as being in a neutral and sane language. + +## Block Structure + +Before executing [state transitions](#state-transitions), the structure of the [block](./data_structures.md#block) must be verified. + +The following block fields are acquired from the network and parsed (i.e. [deserialized](./data_structures.md#serialization)). If they cannot be parsed, the block is ignored but is not explicitly considered invalid by consensus rules. Further implications of ignoring a block are found in the [networking spec](./networking.md). + +1. [block.header](./data_structures.md#header) +1. [block.availableDataHeader](./data_structures.md#availabledataheader) +1. [block.lastCommit](./data_structures.md#commit) + +If the above fields are parsed successfully, the available data `block.availableData` is acquired in erasure-coded form as [a list of share rows](./networking.md#availabledata), then parsed. If it cannot be parsed, the block is ignored but not explicitly invalid, as above. + +### `block.header` + +The [block header](./data_structures.md#header) `block.header` (`header` for short) is the first thing that is downloaded from the new block, and commits to everything inside the block in some way. For previous block `prev` (if `prev` is not known, then the block is ignored), and previous block header `prev.header`, the following checks must be `true`: + +`availableDataOriginalSquareSize` is computed as described [here](./data_structures.md#header). + +1. `header.height` == `prev.header.height + 1`. +1. `header.timestamp` > `prev.header.timestamp`. +1. `header.lastHeaderHash` == the [header hash](./data_structures.md#header) of `prev`. +1. `header.lastCommitHash` == the [hash](./data_structures.md#hashing) of `lastCommit`. +1. `header.consensusHash` == the value computed [here](./data_structures.md#consensus-parameters). +1. `header.stateCommitment` == the root of the state, computed [with the application of all state transitions in this block](#state-transitions). +1. `availableDataOriginalSquareSize` <= [`AVAILABLE_DATA_ORIGINAL_SQUARE_MAX`](#constants). +1. `header.availableDataRoot` == the [Merkle root](./data_structures.md#binary-merkle-tree) of the tree with the row and column roots of `block.availableDataHeader` as leaves. +1. `header.proposerAddress` == the [leader](#leader-selection) for `header.height`. + +### `block.availableDataHeader` + +The [available data header](./data_structures.md#availabledataheader) `block.availableDataHeader` (`availableDataHeader` for short) is then processed. This commits to the available data, which is only downloaded after the [consensus commit](#blocklastcommit) is processed. The following checks must be `true`: + +1. Length of `availableDataHeader.rowRoots` == `availableDataOriginalSquareSize * 2`. +1. Length of `availableDataHeader.colRoots` == `availableDataOriginalSquareSize * 2`. +1. The length of each element in `availableDataHeader.rowRoots` and `availableDataHeader.colRoots` must be [`32`](./data_structures.md#hashing). + +### `block.lastCommit` + +The last [commit](./data_structures.md#commit) `block.lastCommit` (`lastCommit` for short) is processed next. This is the Tendermint commit (i.e. polka of votes) _for the previous block_. For previous block `prev` and previous block header `prev.header`, the following checks must be `true`: + +1. `lastCommit.height` == `prev.header.height`. +1. `lastCommit.round` >= `1`. +1. `lastCommit.headerHash` == the [header hash](./data_structures.md#header) of `prev`. +1. Length of `lastCommit.signatures` <= [`MAX_VALIDATORS`](#constants). +1. Each of `lastCommit.signatures` must be a valid [CommitSig](./data_structures.md#commitsig) +1. The sum of the votes for `prev` in `lastCommit` must be at least 2/3 (rounded up) of the voting power of `prev`'s next validator set. + +### `block.availableData` + +The block's [available data](./data_structures.md#availabledata) (analogous to transactions in contemporary blockchain designs) `block.availableData` (`availableData` for short) is finally processed. The [list of share rows](./networking.md#availabledata) is parsed into the [actual data structures](./data_structures.md#availabledata) using the reverse of [the process to encode available data into shares](./data_structures.md#arranging-available-data-into-shares); if parsing fails here, the block is invalid. + +Once parsed, the following checks must be `true`: + +1. The commitments of the [erasure-coded extended](./data_structures.md#2d-reed-solomon-encoding-scheme) `availableData` must match those in `header.availableDataHeader`. Implicitly, this means that both rows and columns must be ordered lexicographically by namespace ID since they are committed to in a [Namespace Merkle Tree](data_structures.md#namespace-merkle-tree). +1. Length of `availableData.intermediateStateRootData` == length of `availableData.transactionData` + length of `availableData.evidenceData` + 2. (Two additional state transitions are the [begin](#begin-block) and [end block](#end-block) implicit transitions.) + +## State Transitions + +Once the basic structure of the block [has been validated](#block-structure), state transitions must be applied to compute the new state and state root. + +For this section, the variable `state` represents the [state tree](./data_structures.md#state), with `state.accounts[k]`, `state.inactiveValidatorSet[k]`, `state.activeValidatorSet[k]`, and `state.delegationSet[k]` being shorthand for the leaf in the state tree in the [accounts, inactive validator set, active validator set, and delegation set subtrees](./data_structures.md#state) with [pre-hashed key](./data_structures.md#state) `k`. E.g. `state.accounts[a]` is shorthand for `state[(ACCOUNTS_SUBTREE_ID << 8*(32-STATE_SUBTREE_RESERVED_BYTES)) | ((-1 >> 8*STATE_SUBTREE_RESERVED_BYTES) & hash(a))]`. + +State transitions are applied in the following order: + +1. [Begin block](#begin-block). +1. [Evidence](#blockavailabledataevidencedata). +1. [Transactions](#blockavailabledatatransactiondata). +1. [End block](#end-block). + +### `block.availableData.evidenceData` + +Evidence is the second set of state transitions that are applied, ahead of [transactions](#blockavailabledatatransactiondata). +Each evidence represents proof of validator misbehavior, and causes a penalty against the validator(s). + +### `block.availableData.transactionData` + +Once [evidence has been processed](#blockavailabledataevidencedata), transactions are applied to the state. Note that _transactions_ mutate the state (essentially, the validator set and minimal balances), while _messages_ do not. See [the architecture documentation](./architecture.md) for more info. + +`block.availableData.transactionData` is simply a list of [WrappedTransaction](./data_structures.md#wrappedtransaction)s. For each wrapped transaction in this list, `wrappedTransaction`, with index `i` (starting from `0`), the following checks must be `true`: + +1. `wrappedTransaction.index` == `i`. + +For `wrappedTransaction`'s [transaction](./data_structures.md#transaction) `transaction`, the following checks must be `true`: + +1. `transaction.signature` must be a [valid signature](./data_structures.md#public-key-cryptography) over `transaction.signedTransactionData`. + +Finally, each `wrappedTransaction` is processed depending on [its transaction type](./data_structures.md#signedtransactiondata). These are specified in the next subsections, where `tx` is short for `transaction.signedTransactionData`, and `sender` is the recovered signing [address](./data_structures.md#address). We will define a few helper functions: + +```py +tipCost(y, z) = y * z +totalCost(x, y, z) = x + tipCost(y, z) +``` + +where `x` above is the amount of coins sent by the transaction authorizer, `y` above is the tip rate set in the transaction, and `z` above is the measure of the block space used by the transaction (i.e. size in bytes). + +Four additional helper functions are defined to manage the [validator queue](./data_structures.md#validator): + +1. `findFromQueue(power)`, which returns the address of the last validator in the [validator queue](./data_structures.md#validator) with voting power greater than or equal to `power`, or `0` if the queue is empty or no validators in the queue have at least `power` voting power. +1. `parentFromQueue(address)`, which returns the address of the parent in the validator queue of the validator with address `address`, or `0` if `address` is not in the queue or is the head of the queue. +1. `validatorQueueInsert`, defined as + +```py +function validatorQueueInsert(validator) + # Insert the new validator into the linked list + parent = findFromQueue(validator.votingPower) + if parent != 0 + if state.accounts[parent].status == AccountStatus.ValidatorBonded + validator.next = state.activeValidatorSet[parent].next + state.activeValidatorSet[parent].next = sender + else + validator.next = state.inactiveValidatorSet[parent].next + state.inactiveValidatorSet[parent].next = sender + else + validator.next = state.validatorQueueHead + state.validatorQueueHead = sender +``` + + +4. `validatorQueueRemove`, defined as + +```py +function validatorQueueRemove(validator, sender) + # Remove existing validator from the linked list + parent = parentFromQueue(sender) + if parent != 0 + if state.accounts[parent].status == AccountStatus.ValidatorBonded + state.activeValidatorSet[parent].next = validator.next + validator.next = 0 + else + state.inactiveValidatorSet[parent].next = validator.next + validator.next = 0 + else + state.validatorQueueHead = validator.next + validator.next = 0 +``` + +Note that light clients cannot perform a linear search through a linked list, and are instead provided logarithmic proofs (e.g. in the case of `parentFromQueue`, a proof to the parent is provided, which should have `address` as its next validator). + +In addition, three helper functions to manage the [message paid list](./data_structures.md#messagepaid): + +1. `findFromMessagePaidList(start)`, which returns the transaction ID of the last transaction in the [message paid list](./data_structures.md#messagepaid) with `finish` greater than `start`, or `0` if the list is empty or no transactions in the list have at least `start` `finish`. +1. `parentFromMessagePaidList(txid)`, which returns the transaction ID of the parent in the message paid list of the transaction with ID `txid`, or `0` if `txid` is not in the list or is the head of the list. +1. `messagePaidListInsert`, defined as + +```py +function messagePaidListInsert(tx, txid) + # Insert the new transaction into the linked list + parent = findFromMessagePaidList(tx.messageStartIndex) + state.messagesPaid[txid].start = tx.messageStartIndex + numShares = ceil(tx.messageSize / SHARE_SIZE) + state.messagesPaid[txid].finish = tx.messageStartIndex + numShares - 1 + if parent != 0 + state.messagesPaid[txid].next = state.messagesPaid[parent].next + state.messagesPaid[parent].next = txid + else + state.messagesPaid[txid].next = state.messagePaidHead + state.messagePaidHead = txid +``` + +We define a helper function to compute [F1 entries](../rationale/distributing_rewards.md): + +```py +function compute_new_entry(reward, power) + if power == 0 + return 0 + return reward // power +``` + +After applying a transaction, the new state state root is computed. + +#### SignedTransactionDataTransfer + +```py +bytesPaid = len(tx) +``` + +The following checks must be `true`: + +1. `tx.type` == [`TransactionType.Transfer`](./data_structures.md#signedtransactiondata). +1. `totalCost(tx.amount, tx.fee.tipRate, bytesPaid)` <= `state.accounts[sender].balance`. +1. `tx.nonce` == `state.accounts[sender].nonce + 1`. + +Apply the following to the state: + +```py +state.accounts[sender].nonce += 1 + +state.accounts[sender].balance -= totalCost(tx.amount, tx.fee.tipRate, bytesPaid) +state.accounts[tx.to].balance += tx.amount + +state.activeValidatorSet.proposerBlockReward += tipCost(bytesPaid) +``` + +#### SignedTransactionDataMsgPayForData + +```py +bytesPaid = len(tx) + tx.messageSize +currentStartFinish = state.messagesPaid[findFromMessagePaidList(tx.messageStartIndex)] +parentStartFinish = state.messagesPaid[parentFromMessagePaidList(findFromMessagePaidList(tx.messageStartIndex))] +``` + +The following checks must be `true`: + +1. `tx.type` == [`TransactionType.MsgPayForData`](./data_structures.md#signedtransactiondata). +1. `totalCost(0, tx.fee.tipRate, bytesPaid)` <= `state.accounts[sender].balance`. +1. `tx.nonce` == `state.accounts[sender].nonce + 1`. +1. The `ceil(tx.messageSize / SHARE_SIZE)` shares starting at index `tx.messageStartIndex` must: + 1. Have namespace ID `tx.messageNamespaceID`. +1. `tx.messageShareCommitment` == computed as described [here](./data_structures.md#signedtransactiondatamsgpayfordata). +1. `parentStartFinish.finish` < `tx.messageStartIndex`. +1. `currentStartFinish.start` == `0` or `currentStartFinish.start` > `tx.messageStartIndex + ceil(tx.messageSize / SHARE_SIZE)`. + +Apply the following to the state: + +```py +state.accounts[sender].nonce += 1 +state.accounts[sender].balance -= totalCost(tx.amount, tx.fee.tipRate, bytesPaid) + +messagePaidListInsert(tx, id(tx)) + +state.activeValidatorSet.proposerBlockReward += tipCost(tx.fee.tipRate, bytesPaid) +``` + +#### SignedTransactionDataCreateValidator + +```py +bytesPaid = len(tx) +``` + +The following checks must be `true`: + +1. `tx.type` == [`TransactionType.CreateValidator`](./data_structures.md#signedtransactiondata). +1. `totalCost(0, tx.fee.tipRate, bytesPaid)` <= `state.accounts[sender].balance`. +1. `tx.nonce` == `state.accounts[sender].nonce + 1`. +1. `tx.commissionRate.denominator > 0`. +1. `tx.commissionRate.numerator <= tx.commissionRate.denominator`. +1. `state.accounts[sender].status` == `AccountStatus.None`. + +Apply the following to the state: + +```py +state.accounts[sender].nonce += 1 +state.accounts[sender].balance -= totalCost(0, tx.fee.tipRate, bytesPaid) +state.accounts[sender].status = AccountStatus.ValidatorQueued + +validator = new Validator +validator.commissionRate = tx.commissionRate +validator.delegatedCount = 0 +validator.votingPower = 0 +validator.pendingRewards = 0 +validator.latestEntry = PeriodEntry(0) +validator.unbondingHeight = 0 +validator.isSlashed = false + +validatorQueueInsert(validator) + +state.inactiveValidatorSet[sender] = validator + +state.activeValidatorSet.proposerBlockReward += tipCost(tx.fee.tipRate, bytesPaid) +``` + +#### SignedTransactionDataBeginUnbondingValidator + +```py +bytesPaid = len(tx) +``` + +The following checks must be `true`: + +1. `tx.type` == [`TransactionType.BeginUnbondingValidator`](./data_structures.md#signedtransactiondata). +1. `totalCost(0, tx.fee.tipRate, bytesPaid)` <= `state.accounts[sender].balance`. +1. `tx.nonce` == `state.accounts[sender].nonce + 1`. +1. `state.accounts[sender].status` == `AccountStatus.ValidatorQueued` or `state.accounts[sender].status` == `AccountStatus.ValidatorBonded`. + +Apply the following to the state: + +```py +state.accounts[sender].nonce += 1 +state.accounts[sender].balance -= totalCost(0, tx.fee.tipRate, bytesPaid) +state.accounts[sender].status = ValidatorStatus.Unbonding + +if state.accounts[sender].status == AccountStatus.ValidatorQueued + validator = state.inactiveValidatorSet[sender] +else if state.accounts[sender].status == AccountStatus.ValidatorBonded + validator = state.activeValidatorSet[sender] + delete state.activeValidatorSet[sender] + +validator.unbondingHeight = block.height + 1 +validator.latestEntry += compute_new_entry(validator.pendingRewards, validator.votingPower) +validator.pendingRewards = 0 + +validatorQueueRemove(validator, sender) + +state.inactiveValidatorSet[sender] = validator + +state.activeValidatorSet.activeVotingPower -= validator.votingPower + +state.activeValidatorSet.proposerBlockReward += tipCost(tx.fee.tipRate, bytesPaid) +``` + +#### SignedTransactionDataUnbondValidator + +```py +bytesPaid = len(tx) +``` + +The following checks must be `true`: + +1. `tx.type` == [`TransactionType.UnbondValidator`](./data_structures.md#signedtransactiondata). +1. `totalCost(0, tx.fee.tipRate, bytesPaid)` <= `state.accounts[sender].balance`. +1. `tx.nonce` == `state.accounts[sender].nonce + 1`. +1. `state.accounts[sender].status` == `AccountStatus.ValidatorUnbonding`. +1. `state.inactiveValidatorSet[sender].unbondingHeight + UNBONDING_DURATION` < `block.height`. + +Apply the following to the state: + +```py +validator = state.inactiveValidatorSet[sender] + +state.accounts[sender].nonce += 1 +state.accounts[sender].balance -= totalCost(0, tx.fee.tipRate, bytesPaid) +state.accounts[sender].status = AccountStatus.ValidatorUnbonded + +state.accounts[sender].balance += validator.commissionRewards + +state.inactiveValidatorSet[sender] = validator + +if validator.delegatedCount == 0 + state.accounts[sender].status = AccountStatus.None + delete state.inactiveValidatorSet[sender] + +state.activeValidatorSet.proposerBlockReward += tipCost(tx.fee.tipRate, bytesPaid) +``` + +#### SignedTransactionDataCreateDelegation + +```py +bytesPaid = len(tx) +``` + +The following checks must be `true`: + +1. `tx.type` == [`TransactionType.CreateDelegation`](./data_structures.md#signedtransactiondata). +1. `totalCost(tx.amount, tx.fee.tipRate, bytesPaid)` <= `state.accounts[sender].balance`. +1. `state.accounts[tx.to].status` == `AccountStatus.ValidatorQueued` or `state.accounts[tx.to].status` == `AccountStatus.ValidatorBonded`. +1. `tx.nonce` == `state.accounts[sender].nonce + 1`. +1. `state.accounts[sender].status` == `AccountStatus.None`. + +Apply the following to the state: + +```py +state.accounts[sender].nonce += 1 +state.accounts[sender].balance -= totalCost(tx.amount, tx.fee.tipRate, bytesPaid) +state.accounts[sender].status = AccountStatus.DelegationBonded + +if state.accounts[tx.to].status == AccountStatus.ValidatorQueued + validator = state.inactiveValidatorSet[tx.to] +else if state.accounts[tx.to].status == AccountStatus.ValidatorBonded + validator = state.activeValidatorSet[tx.to] + +delegation = new Delegation +delegation.status = DelegationStatus.Bonded +delegation.validator = tx.to +delegation.stakedBalance = tx.amount +delegation.beginEntry = validator.latestEntry +delegation.endEntry = PeriodEntry(0) +delegation.unbondingHeight = 0 + +validator.latestEntry += compute_new_entry(validator.pendingRewards, validator.votingPower) +validator.pendingRewards = 0 +validator.delegatedCount += 1 +validator.votingPower += tx.amount + +# Update the validator in the linked list by first removing then inserting +validatorQueueRemove(validator, delegation.validator) +validatorQueueInsert(validator) + +state.delegationSet[sender] = delegation + +if state.accounts[tx.to].status == AccountStatus.ValidatorQueued + state.inactiveValidatorSet[tx.to] = validator +else if state.accounts[tx.to].status == AccountStatus.ValidatorBonded + state.activeValidatorSet[tx.to] = validator + state.activeValidatorSet.activeVotingPower += tx.amount + +state.activeValidatorSet.proposerBlockReward += tipCost(tx.fee.tipRate, bytesPaid) +``` + +#### SignedTransactionDataBeginUnbondingDelegation + +```py +bytesPaid = len(tx) +``` + +The following checks must be `true`: + +1. `tx.type` == [`TransactionType.BeginUnbondingDelegation`](./data_structures.md#signedtransactiondata). +1. `totalCost(0, tx.fee.tipRate, bytesPaid)` <= `state.accounts[sender].balance`. +1. `tx.nonce` == `state.accounts[sender].nonce + 1`. +1. `state.accounts[sender].status` == `AccountStatus.DelegationBonded`. + +Apply the following to the state: + +```py +state.accounts[sender].nonce += 1 +state.accounts[sender].balance -= totalCost(0, tx.fee.tipRate, bytesPaid) +state.accounts[sender].status = AccountStatus.DelegationUnbonding + +delegation = state.delegationSet[sender] + +if state.accounts[delegation.validator].status == AccountStatus.ValidatorQueued || + state.accounts[delegation.validator].status == AccountStatus.ValidatorUnbonding || + state.accounts[delegation.validator].status == AccountStatus.ValidatorUnbonded + validator = state.inactiveValidatorSet[delegation.validator] +else if state.accounts[delegation.validator].status == AccountStatus.ValidatorBonded + validator = state.activeValidatorSet[delegation.validator] + +delegation.status = DelegationStatus.Unbonding +delegation.endEntry = validator.latestEntry +delegation.unbondingHeight = block.height + 1 + +validator.latestEntry += compute_new_entry(validator.pendingRewards, validator.votingPower) +validator.pendingRewards = 0 +validator.delegatedCount -= 1 +validator.votingPower -= delegation.stakedBalance + +# Update the validator in the linked list by first removing then inserting +# Only do this if the validator is actually in the queue (i.e. bonded or queued) +if state.accounts[delegation.validator].status == AccountStatus.ValidatorBonded || + state.accounts[delegation.validator].status == AccountStatus.ValidatorQueued + validatorQueueRemove(validator, delegation.validator) + validatorQueueInsert(validator) + +state.delegationSet[sender] = delegation + +if state.accounts[delegation.validator].status == AccountStatus.ValidatorQueued || + state.accounts[delegation.validator].status == AccountStatus.ValidatorUnbonding || + state.accounts[delegation.validator].status == AccountStatus.ValidatorUnbonded + state.inactiveValidatorSet[delegation.validator] = validator +else if state.accounts[delegation.validator].status == AccountStatus.ValidatorBonded + state.activeValidatorSet[delegation.validator] = validator + state.activeValidatorSet.activeVotingPower -= delegation.stakedBalance + +state.activeValidatorSet.proposerBlockReward += tipCost(tx.fee.tipRate, bytesPaid) +``` + +#### SignedTransactionDataUnbondDelegation + +```py +bytesPaid = len(tx) +``` + +The following checks must be `true`: + +1. `tx.type` == [`TransactionType.UnbondDelegation`](./data_structures.md#signedtransactiondata). +1. `totalCost(0, bytesPaid)` <= `state.accounts[sender].balance`. +1. `tx.nonce` == `state.accounts[sender].nonce + 1`. +1. `state.accounts[sender].status` == `AccountStatus.DelegationUnbonding`. +1. `state.delegationSet[sender].unbondingHeight + UNBONDING_DURATION` < `block.height`. + +Apply the following to the state: + +```py +delegation = state.accounts[sender].delegationInfo + +state.accounts[sender].nonce += 1 +state.accounts[sender].balance -= totalCost(0, tx.fee.tipRate, bytesPaid) +state.accounts[sender].status = None + +# Return the delegated stake +state.accounts[sender].balance += delegation.stakedBalance +# Also disperse rewards (commission has already been levied) +state.accounts[sender].balance += delegation.stakedBalance * (delegation.endEntry - delegation.beginEntry) + +if state.accounts[delegation.validator].status == AccountStatus.ValidatorQueued || + state.accounts[delegation.validator].status == AccountStatus.ValidatorUnbonding + state.accounts[delegation.validator].status == AccountStatus.ValidatorUnbonded + validator = state.inactiveValidatorSet[delegation.validator] +else if state.accounts[delegation.validator].status == AccountStatus.ValidatorBonded + validator = state.activeValidatorSet[delegation.validator] + +if validator.delegatedCount == 0 && + state.accounts[delegation.validator].status == AccountStatus.ValidatorUnbonded + state.accounts[delegation.validator].status = AccountStatus.None + delete state.inactiveValidatorSet[delegation.validator] + +delete state.accounts[sender].delegationInfo + +state.activeValidatorSet.proposerBlockReward += tipCost(tx.fee.tipRate, bytesPaid) +``` + +#### SignedTransactionDataBurn + +```py +bytesPaid = len(tx) +``` + +The following checks must be `true`: + +1. `tx.type` == [`TransactionType.Burn`](./data_structures.md#signedtransactiondata). +1. `totalCost(tx.amount, bytesPaid)` <= `state.accounts[sender].balance`. +1. `tx.nonce` == `state.accounts[sender].nonce + 1`. + +Apply the following to the state: + +```py +state.accounts[sender].nonce += 1 +state.accounts[sender].balance -= totalCost(tx.amount, tx.fee.tipRate, bytesPaid) + +state.activeValidatorSet.proposerBlockReward += tipCost(tx.fee.tipRate, bytesPaid) +``` + +#### SignedTransactionRedelegateCommission + +```py +bytesPaid = len(tx) +``` + +The following checks must be `true`: + +1. `tx.type` == [`TransactionType.RedelegateCommission`](./data_structures.md#signedtransactiondata). +1. `totalCost(0, tx.fee.tipRate, bytesPaid)` <= `state.accounts[sender].balance`. +1. `tx.nonce` == `state.accounts[sender].nonce + 1`. +1. `state.accounts[tx.to].status` == `AccountStatus.DelegationBonded`. +1. `state.accounts[sender].status` == `AccountStatus.ValidatorBonded`. + +Apply the following to the state: + +```py +state.accounts[sender].nonce += 1 +state.accounts[sender].balance -= totalCost(0, tx.fee.tipRate, bytesPaid) + +delegation = state.delegationSet[tx.to] +validator = state.activeValidatorSet[delegation.validator] + +# Force-redelegate pending rewards for delegation +pendingRewards = delegation.stakedBalance * (validator.latestEntry - delegation.beginEntry) +delegation.stakedBalance += pendingRewards +delegation.beginEntry = validator.latestEntry + +validator.latestEntry += compute_new_entry(validator.pendingRewards, validator.votingPower) +validator.pendingRewards = 0 + +# Assign pending commission rewards to delegation +commissionRewards = validator.commissionRewards +delegation.stakedBalance += commissionRewards +validator.commissionRewards = 0 + +# Update voting power +validator.votingPower += pendingRewards + commissionRewards +state.activeValidatorSet.activeVotingPower += pendingRewards + commissionRewards + +state.delegationSet[tx.to] = delegation +state.activeValidatorSet[delegation.validator] = validator + +state.activeValidatorSet.proposerBlockReward += tipCost(tx.fee.tipRate, bytesPaid) +``` + +#### SignedTransactionRedelegateReward + +```py +bytesPaid = len(tx) +``` + +The following checks must be `true`: + +1. `tx.type` == [`TransactionType.RedelegateReward`](./data_structures.md#signedtransactiondata). +1. `totalCost(0, tx.fee.tipRate, bytesPaid)` <= `state.accounts[sender].balance`. +1. `tx.nonce` == `state.accounts[sender].nonce + 1`. +1. `state.accounts[sender].status` == `AccountStatus.DelegationBonded`. +1. `state.accounts[state.delegationSet[sender].validator].status` == `AccountStatus.ValidatorBonded`. + +Apply the following to the state: + +```py +state.accounts[sender].nonce += 1 +state.accounts[sender].balance -= totalCost(0, tx.fee.tipRate, bytesPaid) + +delegation = state.delegationSet[sender] +validator = state.activeValidatorSet[delegation.validator] + +# Redelegate pending rewards for delegation +pendingRewards = delegation.stakedBalance * (validator.latestEntry - delegation.beginEntry) +delegation.stakedBalance += pendingRewards +delegation.beginEntry = validator.latestEntry + +validator.latestEntry += compute_new_entry(validator.pendingRewards, validator.votingPower) +validator.pendingRewards = 0 + +# Update voting power +validator.votingPower += pendingRewards +state.activeValidatorSet.activeVotingPower += pendingRewards + +state.delegationSet[sender] = delegation +state.activeValidatorSet[delegation.validator] = validator + +state.activeValidatorSet.proposerBlockReward += tipCost(tx.fee.tipRate, bytesPaid) +``` + +#### Begin Block + +At the beginning of the block, rewards are distributed to the block proposer. + +Apply the following to the state: + +```py +proposer = state.activeValidatorSet[block.header.proposerAddress] + +# Compute block subsidy and save to state for use in end block. +rewardFactor = (TARGET_ANNUAL_ISSUANCE * BLOCK_TIME) / (SECONDS_PER_YEAR * sqrt(GENESIS_COIN_COUNT)) +blockReward = rewardFactor * sqrt(state.activeValidatorSet.activeVotingPower) +state.activeValidatorSet.proposerBlockReward = blockReward + +# Save proposer's initial voting power to state for use in end block. +state.activeValidatorSet.proposerInitialVotingPower = proposer.votingPower + +state.activeValidatorSet[block.header.proposerAddress] = proposer +``` + +#### End Block + +Apply the following to the state: + +```py +account = state.accounts[block.header.proposerAddress] + +if account.status == AccountStatus.ValidatorUnbonding + account.status == AccountStatus.ValidatorUnbonded + proposer = state.inactiveValidatorSet[block.header.proposerAddress] +else if account.status == AccountStatus.ValidatorBonded + proposer = state.activeValidatorSet[block.header.proposerAddress] + +# Flush the outstanding pending rewards. +proposer.latestEntry += compute_new_entry(proposer.pendingRewards, proposer.votingPower) +proposer.pendingRewards = 0 + +blockReward = state.activeValidatorSet.proposerBlockReward +commissionReward = proposer.commissionRate.numerator * blockReward // proposer.commissionRate.denominator +proposer.commissionRewards += commissionReward +proposer.pendingRewards += blockReward - commissionReward + +# Even though the voting power hasn't changed yet, we consider this a period change. +proposer.latestEntry += compute_new_entry(proposer.pendingRewards, state.activeValidatorSet.proposerInitialVotingPower) +proposer.pendingRewards = 0 + +if account.status == AccountStatus.ValidatorUnbonding + account.status == AccountStatus.ValidatorUnbonded + state.inactiveValidatorSet[block.header.proposerAddress] = proposer +else if account.status == AccountStatus.ValidatorBonded + state.activeValidatorSet[block.header.proposerAddress] = proposer +``` + +At the end of a block, the top `MAX_VALIDATORS` validators by voting power with voting power _greater than_ zero are or become active (bonded). For newly-bonded validators, the entire validator object is moved to the active validators subtree and their status is changed to bonded. For previously-bonded validators that are no longer in the top `MAX_VALIDATORS` validators begin unbonding. + +Bonding validators is simply setting their status to `AccountStatus.ValidatorBonded`. The logic for validator unbonding is found [here](#signedtransactiondatabeginunbondingvalidator), minus transaction sender updates (nonce, balance, and fee). + +Finally, the state subtree with ID [`MESSAGE_PAID_SUBTREE_ID`](#reserved-state-subtree-ids) is deleted. + +This end block implicit state transition is a single state transition, and [only has a single intermediate state root](#blockavailabledata) associated with it. diff --git a/specs/src/specs/data_structures.md b/specs/src/specs/data_structures.md new file mode 100644 index 0000000000..486da11edf --- /dev/null +++ b/specs/src/specs/data_structures.md @@ -0,0 +1,994 @@ +# Data Structures + +- [Data Structures Overview](#data-structures-overview) +- [Type Aliases](#type-aliases) +- [Blockchain Data Structures](#blockchain-data-structures) + - [Block](#block) + - [Header](#header) + - [AvailableDataHeader](#availabledataheader) + - [AvailableData](#availabledata) + - [Commit](#commit) + - [Timestamp](#timestamp) + - [HashDigest](#hashdigest) + - [TransactionFee](#transactionfee) + - [Address](#address) + - [CommitSig](#commitsig) + - [Signature](#signature) +- [ConsensusVersion](#consensusversion) +- [Serialization](#serialization) +- [Hashing](#hashing) +- [Public-Key Cryptography](#public-key-cryptography) +- [Merkle Trees](#merkle-trees) + - [Binary Merkle Tree](#binary-merkle-tree) + - [BinaryMerkleTreeInclusionProof](#binarymerkletreeinclusionproof) + - [Namespace Merkle Tree](#namespace-merkle-tree) + - [NamespaceMerkleTreeInclusionProof](#namespacemerkletreeinclusionproof) + - [Sparse Merkle Tree](#sparse-merkle-tree) + - [SparseMerkleTreeInclusionProof](#sparsemerkletreeinclusionproof) +- [Erasure Coding](#erasure-coding) + - [Reed-Solomon Erasure Coding](#reed-solomon-erasure-coding) + - [2D Reed-Solomon Encoding Scheme](#2d-reed-solomon-encoding-scheme) + - [Share](#share) + - [Arranging Available Data Into Shares](#arranging-available-data-into-shares) +- [Available Data](#available-data) + - [TransactionData](#transactiondata) + - [WrappedTransaction](#wrappedtransaction) + - [Transaction](#transaction) + - [SignedTransactionData](#signedtransactiondata) + - [SignedTransactionDataTransfer](#signedtransactiondatatransfer) + - [SignedTransactionDataMsgPayForData](#signedtransactiondatamsgpayfordata) + - [SignedTransactionDataCreateValidator](#signedtransactiondatacreatevalidator) + - [SignedTransactionDataBeginUnbondingValidator](#signedtransactiondatabeginunbondingvalidator) + - [SignedTransactionDataUnbondValidator](#signedtransactiondataunbondvalidator) + - [SignedTransactionDataCreateDelegation](#signedtransactiondatacreatedelegation) + - [SignedTransactionDataBeginUnbondingDelegation](#signedtransactiondatabeginunbondingdelegation) + - [SignedTransactionDataUnbondDelegation](#signedtransactiondataunbonddelegation) + - [SignedTransactionDataBurn](#signedtransactiondataburn) + - [SignedTransactionRedelegateCommission](#signedtransactionredelegatecommission) + - [SignedTransactionRedelegateReward](#signedtransactionredelegatereward) + - [IntermediateStateRootData](#intermediatestaterootdata) + - [WrappedIntermediateStateRoot](#wrappedintermediatestateroot) + - [IntermediateStateRoot](#intermediatestateroot) + - [EvidenceData](#evidencedata) + - [Evidence](#evidence) + - [PublicKey](#publickey) + - [Vote](#vote) + - [MessageData](#messagedata) + - [Message](#message) +- [State](#state) + - [StateElement](#stateelement) + - [Account](#account) + - [Delegation](#delegation) + - [Validator](#validator) + - [ActiveValidatorCount](#activevalidatorcount) + - [ActiveVotingPower](#activevotingpower) + - [ProposerBlockReward](#proposerblockreward) + - [ProposerInitialVotingPower](#proposerinitialvotingpower) + - [ValidatorQueueHead](#validatorqueuehead) + - [PeriodEntry](#periodentry) + - [Decimal](#decimal) + - [MessagePaid](#messagepaid) + - [MessagePaidHead](#messagepaidhead) +- [Consensus Parameters](#consensus-parameters) + +## Data Structures Overview + +![fig: Block data structures.](./figures/block_data_structures.svg) + +## Type Aliases + +| name | type | +|-----------------------------|-----------------------------| +| [`Address`](#address) | `byte[32]` | +| `Amount` | `uint64` | +| `Graffiti` | `byte[MAX_GRAFFITI_BYTES]` | +| [`HashDigest`](#hashdigest) | `byte[32]` | +| `Height` | `int64` | +| `NamespaceID` | `byte[NAMESPACE_ID_BYTES]` | +| `Nonce` | `uint64` | +| `Round` | `int32` | +| `StateSubtreeID` | `byte` | +| [`Timestamp`](#timestamp) | `google.protobuf.Timestamp` | +| `VotingPower` | `uint64` | + +## Blockchain Data Structures + +### Block + +Blocks are the top-level data structure of the Celestia blockchain. + +| name | type | description | +|-----------------------|---------------------------------------------|-----------------------------------------------------------------------| +| `header` | [Header](#header) | Block header. Contains primarily identification info and commitments. | +| `availableDataHeader` | [AvailableDataHeader](#availabledataheader) | Header of available data. Contains commitments to erasure-coded data. | +| `availableData` | [AvailableData](#availabledata) | Data that is erasure-coded for availability. | +| `lastCommit` | [Commit](#commit) | Previous block's Tendermint commit. | + +### Header + +Block header, which is fully downloaded by both full clients and light clients. + +| name | type | description | +|-----------------------------------|---------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `version` | [ConsensusVersion](#consensusversion) | The consensus version struct. | +| `chainID` | `string` | The `CHAIN_ID`. | +| `height` | [Height](#type-aliases) | Block height. The genesis block is at height `1`. | +| `timestamp` | [Timestamp](#timestamp) | Timestamp of this block. | +| `lastHeaderHash` | [HashDigest](#hashdigest) | Previous block's header hash. | +| `lastCommitHash` | [HashDigest](#hashdigest) | Previous block's Tendermint commit hash. | +| `consensusHash` | [HashDigest](#hashdigest) | Hash of [consensus parameters](#consensus-parameters) for this block. | +| `stateCommitment` | [HashDigest](#hashdigest) | The [state root](#state) after this block's transactions are applied. | +| `availableDataOriginalSharesUsed` | `uint64` | The number of shares used in the [original data square](#arranging-available-data-into-shares) that are not [tail padding](./consensus.md#reserved-namespace-ids). | +| `availableDataRoot` | [HashDigest](#hashdigest) | Root of [commitments to erasure-coded data](#availabledataheader). | +| `proposerAddress` | [Address](#address) | Address of this block's proposer. | + +The size of the [original data square](#arranging-available-data-into-shares), `availableDataOriginalSquareSize`, isn't explicitly declared in the block header. Instead, it is implicitly computed as the smallest power of 2 whose square is at least `availableDataOriginalSharesUsed` (in other words, the smallest power of 4 that is at least `availableDataOriginalSharesUsed`). + +The header hash is the [hash](#hashing) of the [serialized](#serialization) header. + +### AvailableDataHeader + +| name | type | description | +|------------|-------------------------------|----------------------------------------| +| `rowRoots` | [HashDigest](#hashdigest)`[]` | Commitments to all erasure-coded data. | +| `colRoots` | [HashDigest](#hashdigest)`[]` | Commitments to all erasure-coded data. | + +The number of row/column roots of the original data [shares](data_structures.md#share) in [square layout](#arranging-available-data-into-shares) for this block. The `availableDataRoot` of the [header](#header) is computed using the compact row and column roots as described [here](#2d-reed-solomon-encoding-scheme). + +The number of row and column roots is each `availableDataOriginalSquareSize * 2`, and must be a power of 2. Note that the minimum `availableDataOriginalSquareSize` is 1 (not 0), therefore the number of row and column roots are each at least 2. + +Implementations can prune rows containing only [tail padding](./consensus.md#reserved-namespace-ids) as they are implicitly available. + +### AvailableData + +Data that is [erasure-coded](#erasure-coding) for [data availability checks](https://arxiv.org/abs/1809.09044). + +| name | type | description | +|-----------------------------|---------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------| +| `transactionData` | [TransactionData](#transactiondata) | Transaction data. Transactions modify the validator set and balances, and pay fees for messages to be included. | +| `intermediateStateRootData` | [IntermediateStateRootData](#intermediatestaterootdata) | Intermediate state roots used for fraud proofs. | +| `evidenceData` | [EvidenceData](#evidencedata) | Evidence used for slashing conditions (e.g. equivocation). | +| `messageData` | [MessageData](#messagedata) | Message data. Messages are app data. | + +### Commit + +| name | type | description | +|--------------|-----------------------------|------------------------------------| +| `height` | [Height](#type-aliases) | Block height. | +| `round` | [Round](#type-aliases) | Round. Incremented on view change. | +| `headerHash` | [HashDigest](#hashdigest) | Header hash of the previous block. | +| `signatures` | [CommitSig](#commitsig)`[]` | List of signatures. | + +### Timestamp + +Timestamp is a [type alias](#type-aliases). + +Celestia uses [`google.protobuf.Timestamp`](https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#google.protobuf.Timestamp) to represent time. + +### HashDigest + +HashDigest is a [type alias](#type-aliases). + +Output of the [hashing](#hashing) function. Exactly 256 bits (32 bytes) long. + +### TransactionFee + +| name | type | description | +|-----------|----------|------------------------------------| +| `tipRate` | `uint64` | The tip rate for this transaction. | + +Abstraction over transaction fees. + +### Address + +Address is a [type alias](#type-aliases). + +Addresses are the [hash](#hashing) [digest](#hashdigest) of the [public key](#publickey). + +Addresses have a length of 32 bytes. + +### CommitSig + +```C++ +enum CommitFlag : uint8_t { + CommitFlagAbsent = 1, + CommitFlagCommit = 2, + CommitFlagNil = 3, +}; +``` + +| name | type | description | +|--------------------|-------------------------|-------------| +| `commitFlag` | `CommitFlag` | | +| `validatorAddress` | [Address](#address) | | +| `timestamp` | [Timestamp](#timestamp) | | +| `signature` | [Signature](#signature) | | + +### Signature + +| name | type | description | +|------|------------|----------------------------------------------------------------------| +| `r` | `byte[32]` | `r` value of the signature. | +| `vs` | `byte[32]` | 1-bit `v` value followed by last 255 bits of `s` value of signature. | + +Output of the [signing](#public-key-cryptography) process. + +## ConsensusVersion + +| name | type | description | +|---------|----------|----------------------| +| `block` | `uint64` | The `VERSION_BLOCK`. | +| `app` | `uint64` | The `VERSION_APP`. | + +## Serialization + +Objects that are committed to or signed over require a canonical serialization. This is done using a deterministic (and thus, bijective) variant of protobuf defined [here](https://github.com/cosmos/cosmos-sdk/blob/master/docs/architecture/adr-027-deterministic-protobuf-serialization.md). + +Note: there are two requirements for a serialization scheme, should this need to be changed: + +1. Must be bijective. +1. Serialization must include the length of dynamic structures (e.g. arrays with variable length). + +## Hashing + +All protocol-level hashing is done using SHA-2-256 as defined in [FIPS 180-4](https://doi.org/10.6028/NIST.FIPS.180-4). SHA-2-256 outputs a digest that is 256 bits (i.e. 32 bytes) long. + +Libraries implementing SHA-2-256 are available in Go () and Rust (). + +Unless otherwise indicated explicitly, objects are first [serialized](#serialization) before being hashed. + +## Public-Key Cryptography + +Consensus-critical data is authenticated using [ECDSA](https://www.secg.org/sec1-v2.pdf), with the curve [secp256k1](https://en.bitcoin.it/wiki/Secp256k1). A highly-optimized library is available in C (), with wrappers in Go () and Rust (). + +[Public keys](#publickey) are encoded in uncompressed form, as the concatenation of the `x` and `y` values. No prefix is needed to distinguish between encoding schemes as this is the only encoding supported. + +Deterministic signatures ([RFC-6979](https://tools.ietf.org/rfc/rfc6979.txt)) should be used when signing, but this is not enforced at the protocol level as it cannot be. + +[Signatures](#signature) are represented as the `r` and `s` (each 32 bytes), and `v` (1-bit) values of the signature. `r` and `s` take on their usual meaning (see: [SEC 1, 4.1.3 Signing Operation](https://www.secg.org/sec1-v2.pdf)), while `v` is used for recovering the public key from a signature more quickly (see: [SEC 1, 4.1.6 Public Key Recovery Operation](https://www.secg.org/sec1-v2.pdf)). Only low-`s` values in signatures are valid (i.e. `s <= secp256k1.n//2`); `s` can be replaced with `-s mod secp256k1.n` during the signing process if it is high. Given this, the first bit of `s` will always be `0`, and can be used to store the 1-bit `v` value. + +`v` represents the parity of the `Y` component of the point, `0` for even and `1` for odd. The `X` component of the point is assumed to always be low, since [the possibility of it being high is negligible](https://bitcoin.stackexchange.com/a/38909). + +Putting it all together, the encoding for signatures is: + + +``` +| 32 bytes || 32 bytes | +[256-bit r value][1-bit v value][255-bit s value] +``` + +This encoding scheme is derived from [EIP 2098: Compact Signature Representation](https://eips.ethereum.org/EIPS/eip-2098). + +## Merkle Trees + +Merkle trees are used to authenticate various pieces of data across the Celestia stack, including transactions, messages, the validator set, etc. This section provides an overview of the different tree types used, and specifies how to construct them. + +### Binary Merkle Tree + +Binary Merkle trees are constructed in the same fashion as described in [Certificate Transparency (RFC-6962)](https://tools.ietf.org/html/rfc6962), except for using [a different hashing function](#hashing). Leaves are hashed once to get leaf node values and internal node values are the hash of the concatenation of their children (either leaf nodes or other internal nodes). + +Nodes contain a single field: + +| name | type | description | +|------|---------------------------|-------------| +| `v` | [HashDigest](#hashdigest) | Node value. | + +The base case (an empty tree) is defined as the [hash](#hashing) of the empty string: + +```C++ +node.v = 0xe3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +``` + +For leaf node `node` of leaf data `d`: + +```C++ +node.v = h(0x00, serialize(d)) +``` + +For internal node `node` with children `l` and `r`: + +```C++ +node.v = h(0x01, l.v, r.v) +``` + +Note that rather than duplicating the last node if there are an odd number of nodes (the [Bitcoin design](https://github.com/bitcoin/bitcoin/blob/5961b23898ee7c0af2626c46d5d70e80136578d3/src/consensus/merkle.cpp#L9-L43)), trees are allowed to be imbalanced. In other words, the height of each leaf may be different. For an example, see Section 2.1.3 of [Certificate Transparency (RFC-6962)](https://tools.ietf.org/html/rfc6962#section-2.1.3). + +Leaves and internal nodes are hashed differently: the one-byte `0x00` is prepended for leaf nodes while `0x01` is prepended for internal nodes. This avoids a second-preimage attack [where internal nodes are presented as leaves](https://en.wikipedia.org/wiki/Merkle_tree#Second_preimage_attack) trees with leaves at different heights. + +#### BinaryMerkleTreeInclusionProof + +| name | type | description | +|------------|-------------------------------|-----------------------------------------------------------------| +| `siblings` | [HashDigest](#hashdigest)`[]` | Sibling hash values, ordered starting from the leaf's neighbor. | + +A proof for a leaf in a [binary Merkle tree](#binary-merkle-tree), as per Section 2.1.1 of [Certificate Transparency (RFC-6962)](https://tools.ietf.org/html/rfc6962#section-2.1.1). + +### Namespace Merkle Tree + +[Shares](#share) in Celestia are associated with a provided _namespace ID_. The Namespace Merkle Tree (NMT) is a variation of the [Merkle Interval Tree](https://eprint.iacr.org/2018/642), which is itself an extension of the [Merkle Sum Tree](https://bitcointalk.org/index.php?topic=845978.0). It allows for compact proofs around the inclusion or exclusion of shares with particular namespace IDs. + +Nodes contain three fields: + +| name | type | description | +|---------|------------------------------|--------------------------------------------------| +| `n_min` | [NamespaceID](#type-aliases) | Min namespace ID in subtree rooted at this node. | +| `n_max` | [NamespaceID](#type-aliases) | Max namespace ID in subtree rooted at this node. | +| `v` | [HashDigest](#hashdigest) | Node value. | + +The base case (an empty tree) is defined as: + +```C++ +node.n_min = 0x0000000000000000 +node.n_max = 0x0000000000000000 +node.v = 0xe3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +``` + +For leaf node `node` of [share](#share) data `d`: + +```C++ +node.n_min = d.namespaceID +node.n_max = d.namespaceID +node.v = h(0x00, d.namespaceID, d.rawData) +``` + +The `namespaceID` message field here is the namespace ID of the leaf, which is a [`NAMESPACE_ID_BYTES`](consensus.md#system-parameters)-long byte array. + +Leaves in an NMT **must** be lexicographically sorted by namespace ID in ascending order. + +For internal node `node` with children `l` and `r`: + +```C++ +node.n_min = min(l.n_min, r.n_min) +if l.n_min == PARITY_SHARE_NAMESPACE_ID + node.n_max = PARITY_SHARE_NAMESPACE_ID +else if r.n_min == PARITY_SHARE_NAMESPACE_ID + node.n_max = l.n_max +else + node.n_max = max(l.n_max, r.n_max) +node.v = h(0x01, l.n_min, l.n_max, l.v, r.n_min, r.n_max, r.v) +``` + +Note that the above snippet leverages the property that leaves are sorted by namespace ID: if `l.n_min` is [`PARITY_SHARE_NAMESPACE_ID`](consensus.md#reserved-state-subtree-ids), so must `{l,r}.n_max`. By construction, either both the min and max namespace IDs of a node will be [`PARITY_SHARE_NAMESPACE_ID`](consensus.md#reserved-state-subtree-ids), or neither will: if `r.n_min` is [`PARITY_SHARE_NAMESPACE_ID`](consensus.md#reserved-state-subtree-ids), so must `r.n_max`. + +For some intuition: the min and max namespace IDs for subtree roots with at least one non-parity leaf (which includes the root of an NMT, as [the right half of an NMT as used in Celestia will be parity shares](#2d-reed-solomon-encoding-scheme)) _ignore_ the namespace ID for the parity leaves. Subtree roots with _only parity leaves_ have their min and max namespace ID set to [`PARITY_SHARE_NAMESPACE_ID`](consensus.md#reserved-state-subtree-ids). This allows for shorter proofs into the tree than if the namespace ID of parity shares was not ignored (which would cause the max namespace ID of the root to always be [`PARITY_SHARE_NAMESPACE_ID`](consensus.md#reserved-state-subtree-ids)). + +A compact commitment can be computed by taking the [hash](#hashing) of the [serialized](#serialization) root node. + +#### NamespaceMerkleTreeInclusionProof + +| name | type | description | +|-----------------|----------------------------------|-----------------------------------------------------------------| +| `siblingValues` | [HashDigest](#hashdigest)`[]` | Sibling hash values, ordered starting from the leaf's neighbor. | +| `siblingMins` | [NamespaceID](#type-aliases)`[]` | Sibling min namespace IDs. | +| `siblingMaxes` | [NamespaceID](#type-aliases)`[]` | Sibling max namespace IDs. | + +When verifying an NMT proof, the root hash is checked by reconstructing the root node `root_node` with the computed `root_node.v` (computed as with a [plain Merkle proof](#binarymerkletreeproof)) and the provided `rootNamespaceIDMin` and `rootNamespaceIDMax` as the `root_node.n_min` and `root_node.n_max`, respectively. + +### Sparse Merkle Tree + +Sparse Merkle Trees (SMTs) are _sparse_, i.e. they contain mostly empty leaves. They can be used as key-value stores for arbitrary data, as each leaf is keyed by its index in the tree. Storage efficiency is achieved through clever use of implicit defaults, avoiding the need to store empty leaves. + +Additional rules are added on top of plain [binary Merkle trees](#binary-merkle-tree): + +1. Default values are given to leaf nodes with empty leaves. +1. While the above rule is sufficient to pre-compute the values of intermediate nodes that are roots of empty subtrees, a further simplification is to extend this default value to all nodes that are roots of empty subtrees. The 32-byte zero, i.e. `0x0000000000000000000000000000000000000000000000000000000000000000`, is used as the default value. This rule takes precedence over the above one. +1. The number of hashing operations can be reduced to be logarithmic in the number of non-empty leaves on average, assuming a uniform distribution of non-empty leaf keys. An internal node that is the root of a subtree that contains exactly one non-empty leaf is replaced by that leaf's leaf node. + +Nodes contain a single field: + +| name | type | description | +|------|---------------------------|-------------| +| `v` | [HashDigest](#hashdigest) | Node value. | + +In the base case, where a sparse Merkle tree has `height = 0`, the root of a tree is defined as the [hash](#hashing) of the empty string: + +```C++ +node.v = 0xe3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +``` + +When a sparse Merkle tree has a height of 0, it can have no leaves, and, therefore, no default value children. The root is then calculated as the hash of the empty string, similar to that of an empty binary Merkle tree. + +For a tree with `height > 0`, the root of an empty tree is defined as the default value: + +```C++ +node.v = 0x0000000000000000000000000000000000000000000000000000000000000000 +``` + +Note that this is in contrast to the base case of the sparse and binary Merkle trees, where the root is the hash of the empty string. When a sparse Merkle tree has a height greater than 0, a new tree instance is composed of default value leaves. Nodes containing only default value children have the default value as well. Applying these rules recursively percolates the default value up to the tree's root. + +For leaf node `node` of leaf data `d` with key `k`: + +```C++ +node.v = h(0x00, k, h(serialize(d))) +``` + +The key of leaf nodes must be prepended, since the index of a leaf node that is not at maximum depth cannot be determined without this information. Leaf values are hashed so that they do not need to be included in full in non-membership proofs. + +For internal node `node` with children `l` and `r`: + +```C++ +node.v = h(0x01, l.v, r.v) +``` + +#### SparseMerkleTreeInclusionProof + +SMTs can further be extended with _compact_ proofs. [Merkle proofs](#verifying-annotated-merkle-proofs) are composed, among other things, of a list of sibling node values. We note that, since nodes that are roots of empty subtrees have known values (the default value), these values do not need to be provided explicitly; it is sufficient to simply identify which siblings in the Merkle branch are roots of empty subtrees, which can be done with one bit per sibling. + +For a Merkle branch of height `h`, an `h`-bit value is appended to the proof. The lowest bit corresponds to the sibling of the leaf node, and each higher bit corresponds to the next parent. A value of `1` indicates that the next value in the list of values provided explicitly in the proof should be used, and a value of `0` indicates that the default value should be used. + +A proof into an SMT is structured as: + +| name | type | description | +|--------------------|-------------------------------|--------------------------------------------------------------------------| +| `depth` | `uint16` | Depth of the leaf node. The root node is at depth `0`. Must be `<= 256`. | +| `siblings` | [HashDigest](#hashdigest)`[]` | Sibling hash values, ordered starting from the leaf's neighbor. | +| `includedSiblings` | `byte[32]` | Bitfield of explicitly included sibling hashes. | + +The `includedSiblings` is ordered by most-significant-byte first, with each byte ordered by most-significant-bit first. The lowest bit corresponds to the leaf node level. + +## Erasure Coding + +In order to enable trust-minimized light clients (i.e. light clients that do not rely on an honest majority of validating state assumption), it is critical that light clients can determine whether the data in each block is _available_ or not, without downloading the whole block itself. The technique used here was formally described in the paper [Fraud and Data Availability Proofs: Maximising Light Client Security and Scaling Blockchains with Dishonest Majorities](https://arxiv.org/abs/1809.09044). + +The remainder of the subsections below specify the [2D Reed-Solomon erasure coding scheme](#2d-reed-solomon-encoding-scheme) used, along with the format of [shares](#share) and how [available data](#available-data) is arranged into shares. + +### Reed-Solomon Erasure Coding + +Note that while data is laid out in a two-dimensional square, rows and columns are erasure coded using a standard one-dimensional encoding. + +Reed-Solomon erasure coding is used as the underlying coding scheme. The parameters are: + +- 16-bit Galois field +- [`availableDataOriginalSquareSize`](#header) original pieces (maximum of [`AVAILABLE_DATA_ORIGINAL_SQUARE_MAX`](./consensus.md#constants)) +- [`availableDataOriginalSquareSize`](#header) parity pieces (maximum of [`AVAILABLE_DATA_ORIGINAL_SQUARE_MAX`](./consensus.md#constants)) (i.e `availableDataOriginalSquareSize * 2` total pieces), for an erasure efficiency of 50%. In other words, any 50% of the pieces from the `availableDataOriginalSquareSize * 2` total pieces are enough to recover the original data. +- [`SHARE_SIZE`](./consensus.md#constants) bytes per piece + +Note that [`availableDataOriginalSquareSize`](#header) may vary each block, and [is decided by the block proposer of that block](./block_proposer.md#deciding-on-a-block-size). [Leopard-RS](https://github.com/catid/leopard) is a C library that implements the above scheme with quasilinear runtime. + +### 2D Reed-Solomon Encoding Scheme + +The 2-dimensional data layout is described in this section. The roots of [NMTs](#namespace-merkle-tree) for each row and column across four quadrants of data in a `2k * 2k` matrix of shares, `Q0` to `Q3` (shown below), must be computed. In other words, `2k` row roots and `2k` column roots must be computed. The row and column roots are stored in the `availableDataCommitments` of the [AvailableDataHeader](#availabledataheader). + +![fig: RS2D encoding: data quadrants.](./figures/rs2d_quadrants.svg) + +The data of `Q0` is the original data, and the remaining quadrants are parity data. Setting `k = availableDataOriginalSquareSize`, the original data first must be [split into shares](#share) and [arranged into a `k * k` matrix](#arranging-available-data-into-shares). Then the parity data can be computed. + +Where `A -> B` indicates that `B` is computed using [erasure coding](#reed-solomon-erasure-coding) from `A`: + +- `Q0 -> Q1` for each row in `Q0` and `Q1` +- `Q0 -> Q2` for each column in `Q0` and `Q2` +- `Q2 -> Q3` for each row in `Q2` and `Q3` + +![fig: RS2D encoding: extending data.](./figures/rs2d_extending.svg) + +As an example, the parity data in the second column of `Q2` (in striped purple) is computed by [extending](#reed-solomon-erasure-coding) the original data in the second column of `Q0` (in solid blue). + +![fig: RS2D encoding: extending a column.](./figures/rs2d_extend.svg) + +Now that all four quadrants of the `2k * 2k` matrix are filled, the row and column roots can be computed. To do so, each row/column is used as the leaves of a [NMT](#namespace-merkle-tree), for which the compact root is computed (i.e. an extra hash operation over the NMT root is used to produce a single [HashDigest](#hashdigest)). In this example, the fourth row root value is computed as the NMT root of the fourth row of `Q0` and the fourth row of `Q1` as leaves. + +![fig: RS2D encoding: a row root.](./figures/rs2d_row.svg) + +Finally, the `availableDataRoot` of the block [Header](#header) is computed as the Merkle root of the [binary Merkle tree](#binary-merkle-tree) with the row and column roots as leaves, in that order. + +![fig: Available data root.](./figures/data_root.svg) + +### Share + +| name | type | description | +|---------------|------------------------------|----------------------------| +| `namespaceID` | [NamespaceID](#type-aliases) | Namespace ID of the share. | +| `rawData` | `byte[SHARE_SIZE]` | Raw share data. | + +A share is a fixed-size data chunk associated with a namespace ID, whose data will be erasure-coded and committed to in [Namespace Merkle trees](#namespace-merkle-tree). + +A share's raw data `rawData` is interpreted differently depending on the namespace ID. + +For shares **with a reserved namespace ID through [`NAMESPACE_ID_MAX_RESERVED`](./consensus.md#constants)**: + +- The first [`NAMESPACE_ID_BYTES`](./consensus.md#constants) of a share's raw data `rawData` is the namespace ID of that share, `namespaceID`. +- The next [`SHARE_RESERVED_BYTES`](./consensus.md#constants) bytes (the `*` in the example layout figure below) is the starting byte of the length of the [canonically serialized](#serialization) first request that starts in the share, or `0` if there is none, as a one-byte big-endian unsigned integer (i.e. canonical serialization is not used). In this example, with a share size of `256` the first byte would be `80` (or `0x50` in hex). +- The remaining [`SHARE_SIZE`](./consensus.md#constants)`-`[`NAMESPACE_ID_BYTES`](./consensus.md#constants)`-`[`SHARE_RESERVED_BYTES`](./consensus.md#constants) bytes are request data. + +![fig: Reserved share.](./figures/share.svg) + +For shares **with a namespace ID above [`NAMESPACE_ID_MAX_RESERVED`](./consensus.md#constants) but below [`PARITY_SHARE_NAMESPACE_ID`](./consensus.md#constants)**: + +- The first [`NAMESPACE_ID_BYTES`](./consensus.md#constants) of a share's raw data `rawData` is the namespace ID of that share, `namespaceID`. +- The remaining [`SHARE_SIZE`](./consensus.md#constants)`-`[`NAMESPACE_ID_BYTES`](./consensus.md#constants) bytes are request data. In other words, the remaining bytes have no special meaning and are simply used to store data. + +For shares **with a namespace ID equal to [`PARITY_SHARE_NAMESPACE_ID`](./consensus.md#constants)** (i.e. parity shares): + +- Bytes carry no special meaning. + +For non-parity shares, if there is insufficient request data to fill the share, the remaining bytes are filled with `0`. + +### Arranging Available Data Into Shares + +The previous sections described how some original data, arranged into a `k * k` matrix, can be extended into a `2k * 2k` matrix and committed to with NMT roots. This section specifies how [available data](#available-data) (which includes [transactions](#transactiondata), [intermediate state roots](#intermediatestaterootdata), [evidence](#evidencedata), and [messages](#messagedata)) is arranged into the matrix in the first place. + +Then, + +1. For each of `transactionData`, `intermediateStateRootData`, and `evidenceData`, [serialize](#serialization): + 1. For each request in the list: + 1. [Serialize](#serialization) the request (individually). + 1. Compute the length of each serialized request, [serialize the length](#serialization), and pre-pend the serialized request with its serialized length. + 1. Split up the length/request pairs into [`SHARE_SIZE`](./consensus.md#constants)`-`[`NAMESPACE_ID_BYTES`](./consensus.md#constants)`-`[`SHARE_RESERVED_BYTES`](./consensus.md#constants)-byte chunks. + 1. Create a [share](#share) out of each chunk. This data has a _reserved_ namespace ID, so the first [`NAMESPACE_ID_BYTES`](./consensus.md#constants)`+`[`SHARE_RESERVED_BYTES`](./consensus.md#constants) bytes for these shares must be [set specially](#share). +1. Concatenate the lists of shares in the order: transactions, intermediate state roots, evidence. + +Note that by construction, each share only has a single namespace, and that the list of concatenated shares is [lexicographically ordered by namespace ID](consensus.md#reserved-namespace-ids). + +These shares are arranged in the [first quadrant](#2d-reed-solomon-encoding-scheme) (`Q0`) of the `availableDataOriginalSquareSize*2 * availableDataOriginalSquareSize*2` available data matrix in _row-major_ order. In the example below, each reserved data element takes up exactly one share. + +![fig: Original data: reserved.](./figures/rs2d_originaldata_reserved.svg) + +Each message in the list `messageData`: + +1. [Serialize](#serialization) the message (individually). +1. Compute the length of each serialized message, [serialize the length](#serialization), and pre-pend the serialized message with its serialized length. +1. Split up the length/message pairs into [`SHARE_SIZE`](./consensus.md#constants)`-`[`NAMESPACE_ID_BYTES`](./consensus.md#constants)-byte chunks. +1. Create a [share](#share) out of each chunk. The first [`NAMESPACE_ID_BYTES`](./consensus.md#constants) bytes for these shares is [set to the namespace ID](#share). + +For each message, it is placed in the available data matrix, with row-major order, as follows: + +1. Place the first share of the message at the next unused location in the matrix, then place the remaining shares in the following locations. + +Transactions [must commit to a Merkle root of a list of hashes](#transaction) that are each guaranteed (assuming the block is valid) to be subtree roots in one or more of the row NMTs. For additional info, see [the rationale document](../rationale/message_block_layout.md) for this section. + +However, with only the rule above, interaction between the block producer and transaction sender may be required to compute a commitment to the message the transaction sender can sign over. To remove interaction, messages can optionally be laid out using a non-interactive default: + +1. Place the first share of the message at the next unused location in the matrix whose column is aligned with the largest power of 2 that is not larger than the message length or [`availableDataOriginalSquareSize`](#header), then place the remaining shares in the following locations **unless** there are insufficient unused locations in the row. +1. If there are insufficient unused locations in the row, place the first share of the message at the first column of the next row. Then place the remaining shares in the following locations. By construction, any message whose length is greater than [`availableDataOriginalSquareSize`](#header) will be placed in this way. + +In the example below, two messages (of lengths 2 and 1, respectively) are placed using the aforementioned default non-interactive rules. + +![fig: Original data: messages.](./figures/rs2d_originaldata_message.svg) + +The non-interactive default rules may introduce empty shares that do not belong to any message (in the example above, the top-right share is empty). These are zeroes with namespace ID equal to the either [`TAIL_TRANSACTION_PADDING_NAMESPACE_ID`](./consensus.md#constants) if between a request with a reserved namespace ID and a message, or the namespace ID of the previous message if succeeded by a message. See the [rationale doc](../rationale/message_block_layout.md) for more info. + +## Available Data + +### TransactionData + +| name | type | description | +|-----------------------|-----------------------------------------------|-------------------------------| +| `wrappedTransactions` | [WrappedTransaction](#wrappedtransaction)`[]` | List of wrapped transactions. | + +#### WrappedTransaction + +Wrapped transactions include additional metadata by the block proposer that is committed to in the [available data matrix](#arranging-available-data-into-shares). + +| name | type | description | +|---------------------|-----------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `index` | `uint64` | Index of this transaction in the list of wrapped transactions. This information is lost when splitting transactions into [fixed-sized shares](#share), and needs to be re-added here for fraud proof support. Allows linking a transaction to an [intermediate state root](#wrappedintermediatestateroot). | +| `transaction` | [Transaction](#transaction) | Actual transaction. | +| `messageStartIndex` | `uint64` | _Optional, only used if transaction pays for a message or padding_. Share index (in row-major order) of first share of message this transaction pays for. Needed for light verification of proper message inclusion. | + +#### Transaction + +| name | type | description | +|-------------------------|-------------------------------------------------|-----------------------------------| +| `signedTransactionData` | [SignedTransactionData](#signedtransactiondata) | Data payload that is signed over. | +| `signature` | [Signature](#signature) | Signature. | + +#### SignedTransactionData + +```C++ +enum TransactionType : uint8_t { + Transfer = 1, + MsgPayForData = 2, + CreateValidator = 3, + BeginUnbondingValidator = 4, + UnbondValidator = 5, + CreateDelegation = 6, + BeginUnbondingDelegation = 7, + UnbondDelegation = 8, + Burn = 9, + RedelegateCommission = 10, + RedelegateReward = 11, +}; +``` + +Signed transaction data comes in a number of types: + +1. [Transfer](#signedtransactiondatatransfer) +1. [MsgPayForData](#signedtransactiondatamsgpayfordata) +1. [CreateValidator](#signedtransactiondatacreatevalidator) +1. [BeginUnbondingValidator](#signedtransactiondatabeginunbondingvalidator) +1. [UnbondValidator](#signedtransactiondataunbondvalidator) +1. [CreateDelegation](#signedtransactiondatacreatedelegation) +1. [BeginUnbondingDelegation](#signedtransactiondatabeginunbondingdelegation) +1. [UnbondDelegation](#signedtransactiondataunbonddelegation) +1. [Burn](#signedtransactiondataburn) +1. [RedelegateCommission](#signedtransactionredelegatecommission) +1. [RedelegateReward](#signedtransactionredelegatereward) + +Common fields are denoted here to avoid repeating descriptions: + +| name | type | description | +|----------|-----------------------------------|----------------------------------------------------------------------------| +| `type` | `TransactionType` | Type of the transaction. Each type indicates a different state transition. | +| `amount` | [Amount](#type-aliases) | Amount of coins to send, in `1u`. | +| `to` | [Address](#address) | Recipient's address. | +| `fee` | [TransactionFee](#transactionfee) | The fee information for this transaction. | +| `nonce` | [Nonce](#type-aliases) | Nonce of sender. | + +##### SignedTransactionDataTransfer + +| name | type | description | +|----------|-----------------------------------|-------------------------------------| +| `type` | `TransactionType` | Must be `TransactionType.Transfer`. | +| `amount` | [Amount](#type-aliases) | | +| `to` | [Address](#address) | | +| `fee` | [TransactionFee](#transactionfee) | | +| `nonce` | [Nonce](#type-aliases) | | + +Transfers `amount` coins to `to`. + +##### SignedTransactionDataMsgPayForData + +| name | type | description | +|--------------------------|-----------------------------------|--------------------------------------------------------------| +| `type` | `TransactionType` | Must be `TransactionType.MsgPayForData`. | +| `fee` | [TransactionFee](#transactionfee) | | +| `nonce` | [Nonce](#type-aliases) | | +| `messageNamespaceID` | [`NamespaceID`](#type-aliases) | Namespace ID of message this transaction pays a fee for. | +| `messageSize` | `uint64` | Size of message this transaction pays a fee for, in `byte`s. | +| `messageShareCommitment` | [HashDigest](#hashdigest) | Commitment to message shares (details below). | + +Pays for the inclusion of a [message](#message) in the same block. + +The commitment to message shares `messageShareCommitment` is a [Merkle root](#binary-merkle-tree) of message share roots. Each message share root is [a subtree root in a row NMT](#arranging-available-data-into-shares). For rationale, see [rationale doc](../rationale/message_block_layout.md). + +##### SignedTransactionDataCreateValidator + +| name | type | description | +|------------------|-----------------------------------|--------------------------------------------| +| `type` | `TransactionType` | Must be `TransactionType.CreateValidator`. | +| `fee` | [TransactionFee](#transactionfee) | | +| `nonce` | [Nonce](#type-aliases) | | +| `commissionRate` | [Decimal](#decimal) | | + +Create a new [Validator](#validator) at this address. + +##### SignedTransactionDataBeginUnbondingValidator + +| name | type | description | +|---------|-----------------------------------|----------------------------------------------------| +| `type` | `TransactionType` | Must be `TransactionType.BeginUnbondingValidator`. | +| `fee` | [TransactionFee](#transactionfee) | | +| `nonce` | [Nonce](#type-aliases) | | + +Begin unbonding the [Validator](#validator) at this address. + +##### SignedTransactionDataUnbondValidator + +| name | type | description | +|---------|-----------------------------------|--------------------------------------------| +| `type` | `TransactionType` | Must be `TransactionType.UnbondValidator`. | +| `fee` | [TransactionFee](#transactionfee) | | +| `nonce` | [Nonce](#type-aliases) | | + +Finish unbonding the [Validator](#validator) at this address. + +##### SignedTransactionDataCreateDelegation + +| name | type | description | +|----------|-----------------------------------|---------------------------------------------| +| `type` | `TransactionType` | Must be `TransactionType.CreateDelegation`. | +| `amount` | [Amount](#type-aliases) | | +| `to` | [Address](#address) | | +| `fee` | [TransactionFee](#transactionfee) | | +| `nonce` | [Nonce](#type-aliases) | | + +Create a new [Delegation](#delegation) of `amount` coins worth of voting power for validator with address `to`. + +##### SignedTransactionDataBeginUnbondingDelegation + +| name | type | description | +|---------|-----------------------------------|-----------------------------------------------------| +| `type` | `TransactionType` | Must be `TransactionType.BeginUnbondingDelegation`. | +| `fee` | [TransactionFee](#transactionfee) | | +| `nonce` | [Nonce](#type-aliases) | | + +Begin unbonding the [Delegation](#delegation) at this address. + +##### SignedTransactionDataUnbondDelegation + +| name | type | description | +|---------|-----------------------------------|---------------------------------------------| +| `type` | `TransactionType` | Must be `TransactionType.UnbondDelegation`. | +| `fee` | [TransactionFee](#transactionfee) | | +| `nonce` | [Nonce](#type-aliases) | | + +Finish unbonding the [Delegation](#delegation) at this address. + +##### SignedTransactionDataBurn + +| name | type | description | +|------------|-----------------------------------|----------------------------------------------| +| `type` | `TransactionType` | Must be `TransactionType.Burn`. | +| `amount` | [Amount](#type-aliases) | | +| `fee` | [TransactionFee](#transactionfee) | | +| `nonce` | [Nonce](#type-aliases) | | +| `graffiti` | [Graffiti](#type-aliases) | Graffiti to indicate the reason for burning. | + +##### SignedTransactionRedelegateCommission + +| name | type | description | +|---------|-----------------------------------|-------------------------------------------------| +| `type` | `TransactionType` | Must be `TransactionType.RedelegateCommission`. | +| `to` | [Address](#address) | | +| `fee` | [TransactionFee](#transactionfee) | | +| `nonce` | [Nonce](#type-aliases) | | + +Assigns validator's pending commission to a delegation. + +##### SignedTransactionRedelegateReward + +| name | type | description | +|---------|-----------------------------------|---------------------------------------------| +| `type` | `TransactionType` | Must be `TransactionType.RedelegateReward`. | +| `fee` | [TransactionFee](#transactionfee) | | +| `nonce` | [Nonce](#type-aliases) | | + +Adds delegation's pending rewards to voting power. + +### IntermediateStateRootData + +| name | type | description | +|---------------------------------|-------------------------------------------------------------------|-------------------------------------------| +| `wrappedIntermediateStateRoots` | [WrappedIntermediateStateRoot](#wrappedintermediatestateroot)`[]` | List of wrapped intermediate state roots. | + +#### WrappedIntermediateStateRoot + +| name | type | description | +|-------------------------|-------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `index` | `uint64` | Index of this intermediate state root in the list of intermediate state roots. This information is lost when splitting intermediate state roots into [fixed-sized shares](#share), and needs to be re-added here for fraud proof support. Allows linking an intermediate state root to a [transaction](#wrappedtransaction). | +| `intermediateStateRoot` | [IntermediateStateRoot](#intermediatestateroot) | Intermediate state root. Used for fraud proofs. | + +#### IntermediateStateRoot + +| name | type | description | +|--------|---------------------------|------------------------------------------------------------------------------------------| +| `root` | [HashDigest](#hashdigest) | Root of intermediate state, which is composed of the global state and the validator set. | + +### EvidenceData + +Wrapper for evidence data. + +| name | type | description | +|-------------|---------------------------|------------------------------------------------| +| `evidences` | [Evidence](#evidence)`[]` | List of evidence used for slashing conditions. | + +#### Evidence + +| name | type | description | +|----------|-------------------------|-------------| +| `pubKey` | [PublicKey](#publickey) | | +| `voteA` | [Vote](#vote) | | +| `voteB` | [Vote](#vote) | | + +#### PublicKey + +| name | type | description | +|------|------------|--------------------------| +| `x` | `byte[32]` | `x` value of public key. | +| `y` | `byte[32]` | `y` value of public key. | + +#### Vote + +```C++ +enum VoteType : uint8_t { + Prevote = 1, + Precommit = 2, +}; +``` + +| name | type | description | +|--------------|---------------------------|-------------| +| `type` | `VoteType` | | +| `height` | [Height](#type-aliases) | | +| `round` | [Round](#type-aliases) | | +| `headerHash` | [HashDigest](#hashdigest) | | +| `timestamp` | [Timestamp](#timestamp) | | +| `signature` | [Signature](#signature) | | + +### MessageData + +| name | type | description | +|------------|-------------------------|-------------------| +| `messages` | [Message](#message)`[]` | List of messages. | + +#### Message + +| name | type | description | +|---------------|------------------------------|-------------------------------| +| `namespaceID` | [NamespaceID](#type-aliases) | Namespace ID of this message. | +| `rawData` | `byte[]` | Raw message bytes. | + +## State + +The state of the Celestia chain is intentionally restricted to containing only account balances and the validator set metadata. One unified [Sparse Merkle Tree](#sparse-merkle-tree) is maintained for the entire chain state, the _state tree_. The root of this tree is committed to in the [block header](#header). + +The state tree is separated into `2**(8*STATE_SUBTREE_RESERVED_BYTES)` subtrees, each of which can be used to store a different component of the state. This is done by slicing off the highest `STATE_SUBTREE_RESERVED_BYTES` bytes from the key and replacing them with the appropriate [reserved state subtree ID](consensus.md#reserved-state-subtree-ids). Reducing the key size within subtrees also reduces the collision resistance of keys by `8*STATE_SUBTREE_RESERVED_BYTES` bits, but this is not an issue due the number of bits removed being small. + +A number of subtrees are maintained: + +1. [Accounts](#account) +1. [Active validator set](#validator) +1. [Inactive validator set](#validator) +1. [Delegation set](#delegation) +1. [Message shares paid for](#message-paid) + +### StateElement + +Data structure for state elements is given below: + +| name | type | description | +|-------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------| +| `key` | `byte[32]` | Keys are byte arrays with size 32. | +| `value` | [Account](#account), [Delegation](#delegation), [Validator](#validator), [ActiveValidatorCount](#activevalidatorcount), [ActiveVotingPower](#activevotingpower), [ProposerBlockReward](#proposerblockreward), [ProposerInitialVotingPower](#proposerinitialvotingpower), [ValidatorQueueHead](#validatorqueuehead), [MessagePaidHead](#messagepaidhead) | `value` can be of different types depending on the state elements listed below. There exists a unique protobuf for different state elements. | + +### Account + +```C++ +enum AccountStatus : uint8_t { + None = 1, + DelegationBonded = 2, + DelegationUnbonding = 3, + ValidatorQueued = 4, + ValidatorBonded = 5, + ValidatorUnbonding = 6, + ValidatorUnbonded = 7, +}; +``` + +| name | type | description | +|-----------|-------------------------|-----------------------------------------------------------------------------------| +| `balance` | [Amount](#type-aliases) | Coin balance. | +| `nonce` | [Nonce](#type-aliases) | Account nonce. Every outgoing transaction from this account increments the nonce. | +| `status` | `AccountStatus` | Validator or delegation status of this account. | + +The `status` of an account indicates weather it is a validator (`AccountStatus.Validator*`), delegating to a validator (`AccountStatus.Delegation*`), or neither (`AccountStatus.None`). Being a validator and delegating are mutually exclusive, and only a single validator can be delegated to. + +Delegations have two statuses: + +1. `DelegationBonded`: This delegation is enabled for a `Queued` _or_ `Bonded` validator. Delegations to a `Queued` validator can be withdrawn immediately, while delegations for a `Bonded` validator must be unbonded first. +1. `DelegationUnbonding`: This delegation is unbonding. It will remain in this status for at least `UNBONDING_DURATION` blocks, and while unbonding may still be slashed. Once the unbonding duration has expired, the delegation can be withdrawn. + +Validators have four statuses: + +1. `ValidatorQueued`: This validator has entered the queue to become an active validator. Once the next validator set transition occurs, if this validator has sufficient voting power (including its own stake and stake delegated to it) to be in the top `MAX_VALIDATORS` validators by voting power, it will become an active, i.e. `ValidatorBonded` validator. Until bonded, this validator can immediately exit the queue. +1. `ValidatorBonded`: This validator is active and bonded. It can propose new blocks and vote on proposed blocks. Once bonded, an active validator must go through an unbonding process until its stake can be freed. +1. `ValidatorUnbonding`: This validator is in the process of unbonding, which can be voluntary (the validator decided to stop being an active validator) or forced (the validator committed a slashable offence and was kicked from the active validator set). Validators will remain in this status for at least `UNBONDING_DURATION` blocks, and while unbonding may still be slashed. +1. `ValidatorUnbonded`: This validator has completed its unbonding and has withdrawn its stake. The validator object will remain in this status until `delegatedCount` reaches zero, at which point it is destroyed. + +In the accounts subtree, accounts (i.e. leaves) are keyed by the [hash](#hashdigest) of their [address](#address). The first byte is then replaced with [`ACCOUNTS_SUBTREE_ID`](./consensus.md#reserved-state-subtree-ids). + +### Delegation + +| name | type | description | +|-------------------|------------------------------|-----------------------------------------------------| +| `validator` | [Address](#address) | The validator being delegating to. | +| `stakedBalance` | [VotingPower](#type-aliases) | Delegated stake, in `4u`. | +| `beginEntry` | [PeriodEntry](#periodentry) | Entry when delegation began. | +| `endEntry` | [PeriodEntry](#periodentry) | Entry when delegation ended (i.e. began unbonding). | +| `unbondingHeight` | [Height](#type-aliases) | Block height delegation began unbonding. | + +Delegation objects represent a delegation. + +In the delegation subtree, delegations are keyed by the [hash](#hashdigest) of their [address](#address). The first byte is then replaced with [`DELEGATIONS_SUBTREE_ID`](./consensus.md#reserved-state-subtree-ids). + +### Validator + +| name | type | description | +|---------------------|------------------------------|----------------------------------------------------------------------------------------| +| `commissionRewards` | `uint64` | Validator's commission rewards, in `1u`. | +| `commissionRate` | [Decimal](#decimal) | Commission rate. | +| `delegatedCount` | `uint32` | Number of accounts delegating to this validator. | +| `votingPower` | [VotingPower](#type-aliases) | Total voting power as staked balance + delegated stake, in `4u`. | +| `pendingRewards` | [Amount](#type-aliases) | Rewards collected so far this period, in `1u`. | +| `latestEntry` | [PeriodEntry](#periodentry) | Latest entry, used for calculating reward distribution. | +| `unbondingHeight` | [Height](#type-aliases) | Block height validator began unbonding. | +| `isSlashed` | `bool` | If this validator has been slashed or not. | +| `slashRate` | [Decimal](#decimal) | _Optional_, only if `isSlashed` is set. Rate at which this validator has been slashed. | +| `next` | [Address](#type-aliases) | Next validator in the queue. Zero if this validator is not in the queue. | + +Validator objects represent all the information needed to be keep track of a validator. + +In the validators subtrees, validators are keyed by the [hash](#hashdigest) of their [address](#address). The first byte is then replaced with [`ACTIVE_VALIDATORS_SUBTREE_ID`](./consensus.md#reserved-state-subtree-ids) for the active validator set or [`INACTIVE_VALIDATORS_SUBTREE_ID`](./consensus.md#reserved-state-subtree-ids) for the inactive validator set. Active validators are bonded, (i.e. `ValidatorBonded`), while inactive validators are not bonded (i.e. `ValidatorUnbonded`). By construction, the validators subtrees will be a subset of a mirror of the [accounts subtree](#account). + +The validator queue (i.e. validators with status `ValidatorQueued`) is a subset of the inactive validator set. This queue is represented as a linked list, with each validator pointing to the `next` validator in the queue, and the head of the linked list stored in [ValidatorQueueHead](#validatorqueuehead). + +### ActiveValidatorCount + +| name | type | description | +|-----------------|----------|------------------------------| +| `numValidators` | `uint32` | Number of active validators. | + +Since the [active validator set](#validator) is stored in a [Sparse Merkle Tree](#sparse-merkle-tree), there is no compact way of proving that the number of active validators exceeds `MAX_VALIDATORS` without keeping track of the number of active validators. The active validator count is stored in the active validators subtree, and is keyed with `0` (i.e. `0x0000000000000000000000000000000000000000000000000000000000000000`), with the first byte replaced with `ACTIVE_VALIDATORS_SUBTREE_ID`. + +### ActiveVotingPower + +| name | type | description | +|---------------|----------|----------------------| +| `votingPower` | `uint64` | Active voting power. | + +Since the [active validator set](#validator) is stored in a [Sparse Merkle Tree](#sparse-merkle-tree), there is no compact way of proving the active voting power. The active voting power is stored in the active validators subtree, and is keyed with `1` (i.e. `0x0000000000000000000000000000000000000000000000000000000000000001`), with the first byte replaced with `ACTIVE_VALIDATORS_SUBTREE_ID`. + +### ProposerBlockReward + +| name | type | description | +|----------|----------|--------------------------------------------------------------------------------| +| `reward` | `uint64` | Total block reward (subsidy + fees) in current block so far. Reset each block. | + +The current block reward for the proposer is kept track of here. This is keyed with `2` (i.e. `0x0000000000000000000000000000000000000000000000000000000000000002`), with the first byte replaced with `ACTIVE_VALIDATORS_SUBTREE_ID`. + +### ProposerInitialVotingPower + +| name | type | description | +|---------------|----------|--------------------------------------------------------------------------| +| `votingPower` | `uint64` | Voting power of the proposer at the start of each block. Set each block. | + +The proposer's voting power at the beginning of the block is kept track of here. This is keyed with `3` (i.e. `0x0000000000000000000000000000000000000000000000000000000000000003`), with the first byte replaced with `ACTIVE_VALIDATORS_SUBTREE_ID`. + +### ValidatorQueueHead + +| name | type | description | +|--------|---------------------|-------------------------------------------------------------------| +| `head` | [Address](#address) | Address of inactive validator at the head of the validator queue. | + +The head of the queue for validators that are waiting to become active validators is stored in the inactive validators subtree, and is keyed with `0` (i.e. `0x0000000000000000000000000000000000000000000000000000000000000000`), with the first byte replaced with `INACTIVE_VALIDATORS_SUBTREE_ID`. + +If the queue is empty, `head` is set to the default value (i.e. the hash of the leaf is [the default value for a Sparse Merkle Tree](#sparse-merkle-tree)). + +### PeriodEntry + +| name | type | description | +|--------------|-------------------------|---------------------------------------------------------------| +| `rewardRate` | [Amount](#type-aliases) | Rewards per unit of voting power accumulated so far, in `1u`. | + +For explanation on entries, see the [reward distribution rationale document](../rationale/distributing_rewards.md). + +### Decimal + +| name | type | description | +|---------------|--------|-----------------------| +| `numerator` | uint64 | Rational numerator. | +| `denominator` | uint64 | Rational denominator. | + +Represents a (potentially) non-integer number. + +### MessagePaid + +| name | type | description | +|----------|---------------------------|-----------------------------------------------------------------------| +| `start` | `uint64` | Share index (in row-major order) of first share paid for (inclusive). | +| `finish` | `uint64` | Share index (in row-major order) of last share paid for (inclusive). | +| `next` | [HashDigest](#hashdigest) | Next transaction ID in the list. | + +### MessagePaidHead + +| name | type | description | +|--------|---------------------------|--------------------------------------------------------------------------| +| `head` | [HashDigest](#hashdigest) | Transaction hash at the head of the list (has the smallest start index). | + +The head of the list of paid message shares is stored in the message share paid subtree, and is keyed with `0` (i.e. `0x0000000000000000000000000000000000000000000000000000000000000000`), with the first byte replaced with `MESSAGE_PAID_SUBTREE_ID`. + +If the paid list is empty, `head` is set to the default value (i.e. the hash of the leaf is [the default value for a Sparse Merkle Tree](#sparse-merkle-tree)). + +## Consensus Parameters + +Various [consensus parameters](consensus.md#system-parameters) are committed to in the block header, such as limits and constants. + +| name | type | description | +|----------------------------------|---------------------------------------|-------------------------------------------| +| `version` | [ConsensusVersion](#consensusversion) | The consensus version struct. | +| `chainID` | `string` | The `CHAIN_ID`. | +| `shareSize` | `uint64` | The `SHARE_SIZE`. | +| `shareReservedBytes` | `uint64` | The `SHARE_RESERVED_BYTES`. | +| `availableDataOriginalSquareMax` | `uint64` | The `AVAILABLE_DATA_ORIGINAL_SQUARE_MAX`. | + +In order to compute the `consensusHash` field in the [block header](#header), the above list of parameters is [hashed](#hashing). diff --git a/specs/src/specs/figures/block_data_structures.dot b/specs/src/specs/figures/block_data_structures.dot new file mode 100644 index 0000000000..f98a4e4c4a --- /dev/null +++ b/specs/src/specs/figures/block_data_structures.dot @@ -0,0 +1,36 @@ +digraph G { + rankdir = "LR"; + node [shape = record]; + + subgraph cluster_block { + label = "block"; + + subgraph cluster_availableheader { + label = "availableDataHeader"; + struct4 [label = "{ | { rowRoots | colRoots } }"]; + } + + subgraph cluster_body { + label = "availableData"; + struct3 [label = "{ | { transactionData | intermediateStateRoots | evidenceData | messageData } }"]; + } + + subgraph cluster_lastcommit { + label = "lastCommit"; + struct2 [label = "{lastCommit}"]; + } + + subgraph cluster_header { + label = "header"; + struct1 [label = "version | chainID | height | timestamp | lastHeaderHash | lastCommitHash | consensusHash | stateCommitment | availableDataOriginalSharesUsed | availableDataRoot | proposerAddress"]; + } + } + + struct1:f5 -> struct2; + struct1:f9 -> struct4 [label = "Merkle root of"]; + struct4:f0 -> struct3 [label = "NMT roots to\nerasure-coded data"]; + + edge [style = invis]; + struct1 -> struct3; + struct1 -> struct4; +} \ No newline at end of file diff --git a/specs/src/specs/figures/block_data_structures.svg b/specs/src/specs/figures/block_data_structures.svg new file mode 100644 index 0000000000..5c0bddf8dc --- /dev/null +++ b/specs/src/specs/figures/block_data_structures.svg @@ -0,0 +1,117 @@ + + + + + + +G + + +cluster_block + +block + + +cluster_availableheader + +availableDataHeader + + +cluster_body + +availableData + + +cluster_lastcommit + +lastCommit + + +cluster_header + +header + + + +struct4 + + + +rowRoots + +colRoots + + + +struct3 + + + +transactionData + +intermediateStateRoots + +evidenceData + +messageData + + + +struct4:f0->struct3 + + +NMT roots to +erasure-coded data + + + +struct2 + +lastCommit + + + +struct1 + +version + +chainID + +height + +timestamp + +lastHeaderHash + +lastCommitHash + +consensusHash + +stateCommitment + +availableDataOriginalSharesUsed + +availableDataRoot + +proposerAddress + + + +struct1:f9->struct4 + + +Merkle root of + + + + + +struct1:f5->struct2 + + + + + diff --git a/specs/src/specs/figures/data_root.svg b/specs/src/specs/figures/data_root.svg new file mode 100644 index 0000000000..824c73c1a6 --- /dev/null +++ b/specs/src/specs/figures/data_root.svg @@ -0,0 +1,39 @@ + + + row roots + + + col roots + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + data root + diff --git a/specs/src/specs/figures/rs2d.svg b/specs/src/specs/figures/rs2d.svg new file mode 100644 index 0000000000..b29c8568d0 --- /dev/null +++ b/specs/src/specs/figures/rs2d.svg @@ -0,0 +1,102 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + r1 + rk + r2k + + c1 + ck + c2k + + + + + + + + + + row roots + + + + + + + + + + col roots + diff --git a/specs/src/specs/figures/rs2d_extend.svg b/specs/src/specs/figures/rs2d_extend.svg new file mode 100644 index 0000000000..aca76ac3a2 --- /dev/null +++ b/specs/src/specs/figures/rs2d_extend.svg @@ -0,0 +1,106 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + r1 + rk + r2k + + c1 + ck + c2k + + + + + + + + + + + + + + + + + + + diff --git a/specs/src/specs/figures/rs2d_extending.svg b/specs/src/specs/figures/rs2d_extending.svg new file mode 100644 index 0000000000..42cf5a00c9 --- /dev/null +++ b/specs/src/specs/figures/rs2d_extending.svg @@ -0,0 +1,120 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + r1 + rk + r2k + + c1 + ck + c2k + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + original data + parity data + parity data + parity data + + + + + diff --git a/specs/src/specs/figures/rs2d_originaldata_message.svg b/specs/src/specs/figures/rs2d_originaldata_message.svg new file mode 100644 index 0000000000..23f2065d77 --- /dev/null +++ b/specs/src/specs/figures/rs2d_originaldata_message.svg @@ -0,0 +1,34 @@ + + + + + + + + + + + + + + + + + + + + + r1 + rk + + c1 + ck + + TXs + ISRs + evi. + + msg1 + msg1 + msg2 + diff --git a/specs/src/specs/figures/rs2d_originaldata_reserved.svg b/specs/src/specs/figures/rs2d_originaldata_reserved.svg new file mode 100644 index 0000000000..5e74465a38 --- /dev/null +++ b/specs/src/specs/figures/rs2d_originaldata_reserved.svg @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + + + + + + r1 + rk + + c1 + ck + + TXs + ISRs + evi. + diff --git a/specs/src/specs/figures/rs2d_quadrants.svg b/specs/src/specs/figures/rs2d_quadrants.svg new file mode 100644 index 0000000000..397d6b6e7f --- /dev/null +++ b/specs/src/specs/figures/rs2d_quadrants.svg @@ -0,0 +1,112 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + r1 + rk + r2k + + c1 + ck + c2k + + + + + + + + + + row roots + + + + + + + + + + col roots + + + + + + + Q0 + Q1 + Q2 + Q3 + diff --git a/specs/src/specs/figures/rs2d_row.svg b/specs/src/specs/figures/rs2d_row.svg new file mode 100644 index 0000000000..16f644c8f6 --- /dev/null +++ b/specs/src/specs/figures/rs2d_row.svg @@ -0,0 +1,100 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + r1 + rk + r2k + + c1 + ck + c2k + + + + + + + + + + + + + + + + + + + diff --git a/specs/src/specs/figures/share.dot b/specs/src/specs/figures/share.dot new file mode 100644 index 0000000000..f8ae292bc5 --- /dev/null +++ b/specs/src/specs/figures/share.dot @@ -0,0 +1,25 @@ +digraph G { + node [shape = record, penwidth = 0]; + + share [label=< + + + + + + + + + + + + + + + + + + +
08980200256
NID*end of tx2len(tx3)tx3zero-padding
+ >]; +} \ No newline at end of file diff --git a/specs/src/specs/figures/share.svg b/specs/src/specs/figures/share.svg new file mode 100644 index 0000000000..02885ccc0e --- /dev/null +++ b/specs/src/specs/figures/share.svg @@ -0,0 +1,36 @@ + + + + + + +G + + + +share + +0 +8 +9 +80 +200 +256 + +NID + +* + +end of tx2 + +len(tx3) + +tx3 + +zero-padding + + + diff --git a/specs/src/specs/index.md b/specs/src/specs/index.md new file mode 100644 index 0000000000..265c473fff --- /dev/null +++ b/specs/src/specs/index.md @@ -0,0 +1,6 @@ +# Specification + +- [Data Structures](./data_structures.md) +- [Consensus](./consensus.md) +- [Block Proposer](./block_proposer.md) +- [Networking](./networking.md) diff --git a/specs/src/specs/networking.md b/specs/src/specs/networking.md new file mode 100644 index 0000000000..73edd84f48 --- /dev/null +++ b/specs/src/specs/networking.md @@ -0,0 +1,122 @@ +# Networking + +- [Wire Format](#wire-format) + - [AvailableData](#availabledata) + - [AvailableDataRow](#availabledatarow) + - [ConsensusProposal](#consensusproposal) + - [MsgWirePayForData](#msgwirepayfordata) +- [Invalid Erasure Coding](#invalid-erasure-coding) + - [ShareProof](#shareproof) + - [BadEncodingFraudProof](#badencodingfraudproof) +- [Invalid State Update](#invalid-state-update) + - [StateFraudProof](#statefraudproof) + +## Wire Format + +### AvailableData + +| name | type | description | +|---------------------|-------------------------------------------|---------------| +| `availableDataRows` | [AvailableDataRow](#availabledatarow)`[]` | List of rows. | + +### AvailableDataRow + +| name | type | description | +|----------|-----------------------------------------|------------------| +| `shares` | [Share](./data_structures.md#share)`[]` | Shares in a row. | + +### ConsensusProposal + +Defined as `ConsensusProposal`: + +```protobuf +{{#include ./proto/consensus.proto:ConsensusProposal}} +``` + +When receiving a new block proposal `proposal` from the network, the following steps are performed in order. _Must_ indicates that peers must be blacklisted (to prevent DoS attacks) and _should_ indicates that the network message can simply be ignored. + +1. `proposal.type` must be a `SignedMsgType`. +1. `proposal.round` is processed identically to Tendermint. +1. `proposal.pol_round` is processed identically to Tendermint. +1. `proposal.header` must be well-formed. +1. `proposal.header.version.block` must be [`VERSION_BLOCK`](./consensus.md#constants). +1. `proposal.header.version.app` must be [`VERSION_APP`](./consensus.md#constants). +1. `proposal.header.height` should be previous known height + 1. +1. `proposal.header.chain_id` must be [`CHAIN_ID`](./consensus.md#constants). +1. `proposal.header.time` is processed identically to Tendermint. +1. `proposal.header.last_header_hash` must be previous block's header hash. +1. `proposal.header.last_commit_hash` must be the previous block's commit hash. +1. `proposal.header.consensus_hash` must be the hash of [consensus parameters](./data_structures.md#header). +1. `proposal.header.state_commitment` must be the state root after applying the previous block's transactions. +1. `proposal.header.available_data_original_shares_used` must be at most [`AVAILABLE_DATA_ORIGINAL_SQUARE_MAX ** 2`](./consensus.md#constants). +1. `proposal.header.available_data_root` must be the [root](./data_structures.md#availabledataheader) of `proposal.da_header`. +1. `proposal.header.proposer_address` must be the [correct leader](./consensus.md#leader-selection). +1. `proposal.da_header` must be well-formed. +1. The number of elements in `proposal.da_header.row_roots` and `proposal.da_header.row_roots` must be equal. +1. The number of elements in `proposal.da_header.row_roots` must be the same as computed [here](./data_structures.md#header). +1. `proposal.proposer_signature` must be a valid [digital signature](./data_structures.md#public-key-cryptography) over the header hash of `proposal.header` that recovers to `proposal.header.proposer_address`. +1. For [full nodes](./node_types.md#node-type-definitions), `proposal.da_header` must be the result of computing the roots of the shares (received separately). +1. For [light nodes](./node_types.md#node-type-definitions), `proposal.da_header` should be sampled from for availability. + +### MsgWirePayForData + +Defined as `MsgWirePayForData`: + +```protobuf +{{#include ./proto/wire.proto:MsgWirePayForData}} +``` + +Accepting a `MsgWirePayForData` into the mempool requires different logic than other transactions in Celestia, since it leverages the paradigm of block proposers being able to malleate transaction data. Unlike [SignedTransactionDataMsgPayForData](./data_structures.md#signedtransactiondatamsgpayfordata) (the canonical data type that is included in blocks and committed to with a data root in the block header), each `MsgWirePayForData` (the over-the-wire representation of the same) has potentially multiple signatures. + +Transaction senders who want to pay for a message will create a [SignedTransactionDataMsgPayForData](./data_structures.md#signedtransactiondatamsgpayfordata) object, `stx`, filling in the `stx.messageShareCommitment` field [based on the non-interactive default rules](../rationale/message_block_layout.md#non-interactive-default-rules) for `k = AVAILABLE_DATA_ORIGINAL_SQUARE_MAX`, then signing it to get a [transaction](./data_structures.md#transaction) `tx`. This process is repeated with successively smaller `k`s, decreasing by powers of 2 until `k * k <= stx.messageSize`. At that point, there would be insufficient shares to include both the message and transaction. Using the rest of the signed transaction data along with the pairs of `(tx.signedTransactionData.messageShareCommitment, tx.signature)`, a `MsgWirePayForData` object is constructed. + +Receiving a `MsgWirePayForData` object from the network follows the reverse process: for each `message_commitment_and_signature`, verify using the [non-interactive default rules](../rationale/message_block_layout.md#non-interactive-default-rules) that the signature is valid. + +## Invalid Erasure Coding + +If a malicious block producer incorrectly computes the 2D Reed-Solomon code for a block's data, a fraud proof for this can be presented. We assume that the light clients have the [AvailableDataHeader](./data_structures.md#availabledataheader) and the [Header](./data_structures.md#header) for each block. Hence, given a [ShareProof](#shareproof), they can verify if the `rowRoot` or `colRoot` specified by `isCol` and `position` commits to the corresponding [Share](./data_structures.md#share). Similarly, given the `height` of a block, they can access all elements within the [AvailableDataHeader](./data_structures.md#availabledataheader) and the [Header](./data_structures.md#header) of the block. + +### ShareProof + +| name | type | description | +|------------|---------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------| +| `share` | [Share](./data_structures.md#share) | The share. | +| `proof` | [NamespaceMerkleTreeInclusionProof](./data_structures.md#namespacemerkletreeinclusionproof) | The Merkle proof of the share in the offending row or column root. | +| `isCol` | `bool` | A Boolean indicating if the proof is from a row root or column root; `false` if it is a row root. | +| `position` | `uint64` | The index of the share in the offending row or column. | + +### BadEncodingFraudProof + +Defined as `BadEncodingFraudProof`: + +```protobuf +{{#include ./proto/types.proto:BadEncodingFraudProof}} +``` + +| name | type | description | +|---------------|---------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------| +| `height` | [Height](./data_structures.md#type-aliases) | Height of the block with the offending row or column. | +| `shareProofs` | [ShareProof](#shareproof)`[]` | The available shares in the offending row or column. | +| `isCol` | `bool` | A Boolean indicating if it is an offending row or column; `false` if it is a row. | +| `position` | `uint64` | The index of the offending row or column in the square. | + +## Invalid State Update + +If a malicious block producer incorrectly computes the state, a fraud proof for this can be presented. We assume that the light clients have the [AvailableDataHeader](./data_structures.md#availabledataheader) and the [Header](./data_structures.md#header) for each block. Hence, given a [ShareProof](#shareproof), they can verify if the `rowRoot` or `colRoot` specified by `isCol` and `position` commits to the corresponding [Share](./data_structures.md#share). Similarly, given the `height` of a block, they can access all elements within the [AvailableDataHeader](./data_structures.md#availabledataheader) and the [Header](./data_structures.md#header) of the block. + +### StateFraudProof + +Defined as `StateFraudProof`: + +```protobuf +{{#include ./proto/types.proto:StateFraudProof}} +``` + +| name | type | description | +|----------------------------|------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `height` | [Height](./data_structures.md#type-aliases) | Height of the block with the intermediate state roots. Subtracting one from `height` gives the height of the block with the transactions. | +| `transactionShareProofs` | [ShareProof](#shareproof)`[]` | `isCol` of type `bool` must be `false`. | +| `isrShareProofs` | [ShareProof](#shareproof)`[]` | `isCol` of type `bool` must be `false`. | +| `index` | `uint64` | Index for connecting the [WrappedIntermediateStateRoot](./data_structures.md#wrappedintermediatestateroot) and [WrappedTransaction](./data_structures.md#wrappedtransaction) after shares are parsed. | +| `intermediateStateElements`| [StateElement](./data_structures.md#stateelement)`[]` | State elements that were changed by the transactions. | +| `stateInclusionProofs` | [SparseMerkleTreeInclusionProof](./data_structures.md#sparsemerkletreeinclusionproof)`[]`| SparseMerkleTree inclusion proofs for the state elements. | diff --git a/specs/src/specs/proto/consensus.proto b/specs/src/specs/proto/consensus.proto new file mode 100644 index 0000000000..1260c77680 --- /dev/null +++ b/specs/src/specs/proto/consensus.proto @@ -0,0 +1,47 @@ +syntax = "proto3"; + +import "google/protobuf/timestamp.proto"; + +import "types.proto"; + +// Define wire types for consensus messages. + +// SignedMsgType is a type of signed message in the consensus. +enum SignedMsgType { + SignedMsgTypeUnknown = 0; + + // Votes + SignedMsgTypePrevote = 1; + SignedMsgTypePrecommit = 2; + + // Proposals + SignedMsgTypeProposal = 32; +} + +// ANCHOR: ConsensusProposal +message ConsensusProposal { + SignedMsgType type = 1; + int32 round = 2; + int32 pol_round = 3; + // 32-byte hash + // Proposed block header + Header header = 4; + AvailableDataHeader da_header = 5; + // 64-byte signature + bytes proposer_signature = 6; +} +// ANCHOR_END: ConsensusProposal + +message ConsensusVote { + SignedMsgType type = 1; + int64 height = 2; + int32 round = 3; + // 32-byte hash + bytes header_hash = 4; + google.protobuf.Timestamp timestamp = 5; + // 32-byte hash + bytes validator_address = 6; + int32 validator_index = 7; + // 64-byte signature + bytes signature = 8; +} diff --git a/specs/src/specs/proto/types.proto b/specs/src/specs/proto/types.proto new file mode 100644 index 0000000000..6e5e03036f --- /dev/null +++ b/specs/src/specs/proto/types.proto @@ -0,0 +1,285 @@ +syntax = "proto3"; + +import "google/protobuf/timestamp.proto"; + +// Define data structures. + +// https://github.com/celestiaorg/celestia-specs/blob/master/specs/data_structures.md#header +message Header { + ConsensusVersion version = 1; + string chain_id = 2; + int64 height = 3; + google.protobuf.Timestamp time = 4; + // 32-byte hash + BlockID last_header_hash = 5; + // 32-byte hash + bytes last_commit_hash = 6; + // 32-byte hash + bytes consensus_hash = 7; + // 32-byte hash + bytes state_commitment = 8; + uint64 available_data_original_shares_used = 9; + // 32-byte hash + bytes available_data_root = 10; + bytes proposer_address = 11; +} + +// https://github.com/celestiaorg/celestia-specs/blob/master/specs/data_structures.md#consensusversion +message ConsensusVersion { + uint64 block = 1; + uint64 app = 2; +} + +// https://github.com/celestiaorg/celestia-specs/blob/master/specs/data_structures.md#availabledataheader +message AvailableDataHeader { + // array of 32-byte hashes + repeated bytes row_roots = 1; + // array of 32-byte hashes + repeated bytes col_roots = 2; +} + +// Protobuf definitions for the contents of state elements + +// https://github.com/celestiaorg/celestia-specs/blob/master/specs/networking.md#account +// AccountStatus +enum AccountStatus : uint8_t { + None = 1, + DelegationBonded = 2, + DelegationUnbonding = 3, + ValidatorQueued = 4, + ValidatorBonded = 5, + ValidatorUnbonding = 6, + ValidatorUnbonded = 7, +} + +// https://github.com/celestiaorg/celestia-specs/blob/master/specs/networking.md#periodentry +message PeriodEntry { + // Rewards per unit of voting power accumulated so far, in 1u + uint64 rewardRate = 1; +} + +// https://github.com/celestiaorg/celestia-specs/blob/master/specs/networking.md#decimal +message Decimal { + // Rational numerator + uint64 numerator = 1; + // Rational denominator + uint64 denominator = 1; +} + +// https://github.com/celestiaorg/celestia-specs/blob/master/specs/networking.md#messagepaid +message MessagePaid { + // Share index (in row-major order) of first share paid for (inclusive) + uint64 start = 1; + // Share index (in row-major order) of last share paid for (inclusive) + uint64 finish = 2; + // Next transaction ID in the list + // 32-byte hash + bytes next = 3; +} + +// Protobuf definitions for the state elements + +// https://github.com/celestiaorg/celestia-specs/blob/master/specs/networking.md#account +message Account { + // Coin balance + uint64 balance = 1; + // Account nonce. Every outgoing transaction from this account increments the nonce. + uint64 nonce = 2; + // Validator or delegation status of this account + AccountStatus status = 3; +} + +// https://github.com/celestiaorg/celestia-specs/blob/master/specs/networking.md#delegation +message Delegation { + // The validator being delegating to + // 32-bytes + bytes validator = 1; + // Delegated stake, in 4u + uint64 stakedBalance = 2; + // Entry when delegation began + PeriodEntry beginEntry = 3; + // Entry when delegation ended (i.e. began unbonding) + PeriodEntry endEntry = 4; + // Block height delegation began unbonding + int64 unbondingHeight = 5; +} + +// https://github.com/celestiaorg/celestia-specs/blob/master/specs/networking.md#validator +message StateValidator { + // Validator's commission rewards, in 1u + uint64 commissionRewards = 1; + // Commission rate + Decimal commissionRate = 2; + // Number of accounts delegating to the validator + uint32 delegatedCount = 3; + // Total voting power as staked balance + delegated stake, in 4u + uint64 votingPower = 4; + // Rewards collected so far this period, in 1u + uint64 pendingRewards = 5; + // Latest entry, used for calculating reward distribution + PeriodEntry latestEntry = 6; + // Block height validator began unbonding + int64 unbondingHeight = 7; + // If this validator has been slashed or not + bool isSlashed = 8; + // Rate at which this validator has been slashed + // slashRate should be zero if isSlashed is false. + Decimal slashRate = 9; + // Next validator in the queue. Zero if this validator is not in the queue + // 32-bytes + bytes next = 10; +} + +// https://github.com/celestiaorg/celestia-specs/blob/master/specs/networking.md#activevalidatorcount +message ActiveValidatorCount { + // Number of active validators + uint32 numValidators = 1; +} + +// https://github.com/celestiaorg/celestia-specs/blob/master/specs/networking.md#activevotingpower +message ActiveVotingPower { + // Active voting power + uint64 votingPower = 1; +} + +// https://github.com/celestiaorg/celestia-specs/blob/master/specs/networking.md#proposerblockreward +message ProposerBlockReward { + // Total block reward (subsidy + fees) in current block so far. Reset each block + uint64 reward = 1; +} + + +// https://github.com/celestiaorg/celestia-specs/blob/master/specs/networking.md#proposerinitialvotingpower +message ProposerInitialVotingPower { + // Voting power of the proposer at the start of each block. Set each block + uint64 votingPower = 1; +} + + +// https://github.com/celestiaorg/celestia-specs/blob/master/specs/networking.md#validatorqueuehead +message ValidatorQueueHead { + // Address of inactive validator at the head of the validator queue + // 32-bytes + bytes head = 1; +} + +// https://github.com/celestiaorg/celestia-specs/blob/master/specs/networking.md#messagepaidhead +message MessagePaidHead { + // Transaction hash at the head of the list (has the smallest start index) + // 32-byte hash + bytes head = 1; +} + +//Protobuf definitions for the contents of fraud proofs + +// https://github.com/celestiaorg/celestia-specs/blob/master/specs/data_structures.md#namespacemerkletreeinclusionproof +message NamespaceMerkleTreeInclusionProof { + // sibling hash values, ordered starting from the leaf's neighbor + // array of 32-byte hashes + repeated bytes siblingValues = 1; + // sibling min namespace IDs + // array of NAMESPACE_ID_BYTES-bytes + repeated bytes siblingMins = 2; + // sibling max namespace IDs + // array of NAMESPACE_ID_BYTES-bytes + repeated bytes siblingMaxes = 3; +} + +// https://github.com/celestiaorg/celestia-specs/blob/master/specs/data_structures.md#sparsemerkletreeinclusionproof +message SparseMerkleTreeInclusionProof { + // depth of the leaf node, must be <= 256 + // The root node is at depth 0. + uint16 depth = 1; + // sibling hash values, ordered starting from the leaf's neighbor + // array of 32-byte hashes + repeated bytes siblings = 2; + // bitfield of explicitly included sibling hashes + // 32-byte + bytes includedSiblings = 3; +} + +// https://github.com/celestiaorg/celestia-specs/blob/master/specs/data_structures.md#share +message Share { + // namespace ID of the share + // NAMESPACE_ID_BYTES-bytes + bytes namespaceID = 1; + // raw share data + // SHARE_SIZE-bytes + bytes rawData = 2; +} + +// https://github.com/celestiaorg/celestia-specs/blob/master/specs/data_structures.md#stateelement +message StateElement { + // key of the state element + // 32-bytes + bytes key = 1; + // key of the state element + // value can be of different types depending on the state element. + // There exists a unique protobuf for different state elements. + oneof value = { + Account account = 2; + Delegation delegation = 3; + StateValidator stateValidator = 4; + ActiveValidatorCount activeValidatorCount = 5; + ActiveVotingPower activeVotingPower = 6; + ProposerBlockReward proposerBlockReward = 7; + ProposerInitialVotingPower proposerInitialVotingPower = 8; + ValidatorQueueHead validatorQueueHead = 9; + MessagePaid messagePaid = 10; + MessagePaidHead messagePaidHead = 11; + } +} + +// https://github.com/celestiaorg/celestia-specs/blob/master/specs/networking.md#shareproof +message ShareProof { + // the share + Share share = 1; + // the Merkle proof of the share in the offending row or column root + NamespaceMerkleTreeInclusionProof proof = 2; + // a Boolean indicating if the Merkle proof is from a row root or column root; false if it is a row root + bool isCol = 3; + // the index of the share in the offending row or column + uint64 position = 4; +} + +//Protobuf definitions for the fraud proofs + +// ANCHOR: BadEncodingFraudProof +// https://github.com/celestiaorg/celestia-specs/blob/master/specs/networking.md#badencodingfraudproof +message BadEncodingFraudProof { + // height of the block with the offending row or column + int64 height = 1; + // the available shares in the offending row or column and their Merkle proofs + // array of ShareProofs + repeated ShareProof shareProofs = 2; + // a Boolean indicating if it is an offending row or column; false if it is a row + bool isCol = 3; + // the index of the offending row or column in the square + uint64 position = 4; +} +// ANCHOR_END: BadEncodingFraudProof + +// ANCHOR: StateFraudProof +// https://github.com/celestiaorg/celestia-specs/blob/master/specs/networking.md#statefraudproof +message StateFraudProof { + // height of the block with the intermediate state roots + // Subtracting one from height gives the height of the block with the transactions. + int64 height = 1; + // shares containing the transactions and their Merkle proofs + // isCol within the ShareProof must be false. + // array of ShareProofs + repeated ShareProof transactionShareProofs = 2; + // shares containing the intermediate state roots and their Merkle proofs + // isCol within the ShareProof must be false. + // array of ShareProofs + repeated ShareProof isrShareProofs = 3; + // index for connecting the WrappedIntermediateStateRoot and WrappedTransaction after shares are parsed + uint64 index = 4; + // state elements that were changed by the transactions + // array of StateElements + repeated StateElement intermediateStateElements = 5; + // sparse Merkle tree inclusion proofs for the state elements + // array of SparseMerkleTreeInclusionProofs + repeated SparseMerkleTreeInclusionProof stateInclusionProofs = 6; +} +// ANCHOR_END: StateFraudProof diff --git a/specs/src/specs/proto/wire.proto b/specs/src/specs/proto/wire.proto new file mode 100644 index 0000000000..a68c6a456c --- /dev/null +++ b/specs/src/specs/proto/wire.proto @@ -0,0 +1,98 @@ +syntax = "proto3"; + +// Define wire messages for transaction types. +// https://github.com/celestiaorg/celestia-specs/blob/master/specs/data_structures.md#signedtransactiondata + +message TransactionFee { + // Tip fee rate (rewarded to block proposer) + uint64 tip_rate = 1; +} + +message Decimal { + // https://github.com/celestiaorg/celestia-specs/issues/120 + uint64 todo = 1; +} + +message MessageCommitmentAndSignature { + // Original square size the share commitment was computed for + uint64 k = 1; + // 32-byte hash: commitment to message shares + bytes share_commitment = 2; + // 64-byte signature a single SignedTransactionPayForMessage + // https://github.com/celestiaorg/celestia-specs/blob/master/specs/data_structures.md#signedtransactiondatamsgpayfordata + bytes signature = 3; +} + +message WireTxTransfer { + uint64 amount = 1; + // 32-byte address + bytes to = 2; + TransactionFee fee = 3; + uint64 nonce = 4; +} + +// ANCHOR: MsgWirePayForData +message MsgWirePayForData { + TransactionFee fee = 1; + uint64 nonce = 2; + // 8-byte namespace ID + bytes message_namespace_id = 3; + uint64 message_size = 4; + bytes message = 5; + repeated MessageCommitmentAndSignature message_commitment_and_signature = 6; +} +// ANCHOR_END: MsgWirePayForData + +message WireTxCreateValidator { + TransactionFee fee = 1; + uint64 nonce = 2; + Decimal commission_rate = 3; +} + +message WireTxBeginUnbondingValidator { + TransactionFee fee = 1; + uint64 nonce = 2; +} + +message WireTxUnbondValidator { + TransactionFee fee = 1; + uint64 nonce = 2; +} + +message WireTxCreateDelegation { + uint64 amount = 1; + // 32-byte address + bytes to = 2; + TransactionFee fee = 3; + uint64 nonce = 4; +} + +message WireTxBeginUnbondingDelegation { + TransactionFee fee = 1; + uint64 nonce = 2; +} + +message WireTxUnbondDelegation { + TransactionFee fee = 1; + uint64 nonce = 2; +} + +message WireTxBurn { + uint64 amount = 1; + TransactionFee fee = 2; + uint64 nonce = 3; + // 32-byte graffiti + bytes graffiti = 4; +} + +message WireTxRedelegateCommission { + // 32-byte address + bytes to = 1; + TransactionFee fee = 2; + uint64 nonce = 3; +} + +message WireTxRedelegateReward { + TransactionFee fee = 1; + uint64 nonce = 2; +} diff --git a/specs/theme/highlight.js b/specs/theme/highlight.js new file mode 100644 index 0000000000..e40b681e2f --- /dev/null +++ b/specs/theme/highlight.js @@ -0,0 +1,1328 @@ +/* + Highlight.js 10.7.2 (00233d63) + License: BSD-3-Clause + Copyright (c) 2006-2021, Ivan Sagalaev +*/ +var hljs=function(){"use strict";function e(t){ +return t instanceof Map?t.clear=t.delete=t.set=()=>{ +throw Error("map is read-only")}:t instanceof Set&&(t.add=t.clear=t.delete=()=>{ +throw Error("set is read-only") +}),Object.freeze(t),Object.getOwnPropertyNames(t).forEach((n=>{var i=t[n] +;"object"!=typeof i||Object.isFrozen(i)||e(i)})),t}var t=e,n=e;t.default=n +;class i{constructor(e){ +void 0===e.data&&(e.data={}),this.data=e.data,this.isMatchIgnored=!1} +ignoreMatch(){this.isMatchIgnored=!0}}function s(e){ +return e.replace(/&/g,"&").replace(//g,">").replace(/"/g,""").replace(/'/g,"'") +}function a(e,...t){const n=Object.create(null);for(const t in e)n[t]=e[t] +;return t.forEach((e=>{for(const t in e)n[t]=e[t]})),n}const r=e=>!!e.kind +;class l{constructor(e,t){ +this.buffer="",this.classPrefix=t.classPrefix,e.walk(this)}addText(e){ +this.buffer+=s(e)}openNode(e){if(!r(e))return;let t=e.kind +;e.sublanguage||(t=`${this.classPrefix}${t}`),this.span(t)}closeNode(e){ +r(e)&&(this.buffer+="")}value(){return this.buffer}span(e){ +this.buffer+=``}}class o{constructor(){this.rootNode={ +children:[]},this.stack=[this.rootNode]}get top(){ +return this.stack[this.stack.length-1]}get root(){return this.rootNode}add(e){ +this.top.children.push(e)}openNode(e){const t={kind:e,children:[]} +;this.add(t),this.stack.push(t)}closeNode(){ +if(this.stack.length>1)return this.stack.pop()}closeAllNodes(){ +for(;this.closeNode(););}toJSON(){return JSON.stringify(this.rootNode,null,4)} +walk(e){return this.constructor._walk(e,this.rootNode)}static _walk(e,t){ +return"string"==typeof t?e.addText(t):t.children&&(e.openNode(t), +t.children.forEach((t=>this._walk(e,t))),e.closeNode(t)),e}static _collapse(e){ +"string"!=typeof e&&e.children&&(e.children.every((e=>"string"==typeof e))?e.children=[e.children.join("")]:e.children.forEach((e=>{ +o._collapse(e)})))}}class c extends o{constructor(e){super(),this.options=e} +addKeyword(e,t){""!==e&&(this.openNode(t),this.addText(e),this.closeNode())} +addText(e){""!==e&&this.add(e)}addSublanguage(e,t){const n=e.root +;n.kind=t,n.sublanguage=!0,this.add(n)}toHTML(){ +return new l(this,this.options).value()}finalize(){return!0}}function g(e){ +return e?"string"==typeof e?e:e.source:null} +const u=/\[(?:[^\\\]]|\\.)*\]|\(\??|\\([1-9][0-9]*)|\\./,h="[a-zA-Z]\\w*",d="[a-zA-Z_]\\w*",f="\\b\\d+(\\.\\d+)?",p="(-?)(\\b0[xX][a-fA-F0-9]+|(\\b\\d+(\\.\\d*)?|\\.\\d+)([eE][-+]?\\d+)?)",m="\\b(0b[01]+)",b={ +begin:"\\\\[\\s\\S]",relevance:0},E={className:"string",begin:"'",end:"'", +illegal:"\\n",contains:[b]},x={className:"string",begin:'"',end:'"', +illegal:"\\n",contains:[b]},v={ +begin:/\b(a|an|the|are|I'm|isn't|don't|doesn't|won't|but|just|should|pretty|simply|enough|gonna|going|wtf|so|such|will|you|your|they|like|more)\b/ +},w=(e,t,n={})=>{const i=a({className:"comment",begin:e,end:t,contains:[]},n) +;return i.contains.push(v),i.contains.push({className:"doctag", +begin:"(?:TODO|FIXME|NOTE|BUG|OPTIMIZE|HACK|XXX):",relevance:0}),i +},y=w("//","$"),N=w("/\\*","\\*/"),R=w("#","$");var _=Object.freeze({ +__proto__:null,MATCH_NOTHING_RE:/\b\B/,IDENT_RE:h,UNDERSCORE_IDENT_RE:d, +NUMBER_RE:f,C_NUMBER_RE:p,BINARY_NUMBER_RE:m, +RE_STARTERS_RE:"!|!=|!==|%|%=|&|&&|&=|\\*|\\*=|\\+|\\+=|,|-|-=|/=|/|:|;|<<|<<=|<=|<|===|==|=|>>>=|>>=|>=|>>>|>>|>|\\?|\\[|\\{|\\(|\\^|\\^=|\\||\\|=|\\|\\||~", +SHEBANG:(e={})=>{const t=/^#![ ]*\// +;return e.binary&&(e.begin=((...e)=>e.map((e=>g(e))).join(""))(t,/.*\b/,e.binary,/\b.*/)), +a({className:"meta",begin:t,end:/$/,relevance:0,"on:begin":(e,t)=>{ +0!==e.index&&t.ignoreMatch()}},e)},BACKSLASH_ESCAPE:b,APOS_STRING_MODE:E, +QUOTE_STRING_MODE:x,PHRASAL_WORDS_MODE:v,COMMENT:w,C_LINE_COMMENT_MODE:y, +C_BLOCK_COMMENT_MODE:N,HASH_COMMENT_MODE:R,NUMBER_MODE:{className:"number", +begin:f,relevance:0},C_NUMBER_MODE:{className:"number",begin:p,relevance:0}, +BINARY_NUMBER_MODE:{className:"number",begin:m,relevance:0},CSS_NUMBER_MODE:{ +className:"number", +begin:f+"(%|em|ex|ch|rem|vw|vh|vmin|vmax|cm|mm|in|pt|pc|px|deg|grad|rad|turn|s|ms|Hz|kHz|dpi|dpcm|dppx)?", +relevance:0},REGEXP_MODE:{begin:/(?=\/[^/\n]*\/)/,contains:[{className:"regexp", +begin:/\//,end:/\/[gimuy]*/,illegal:/\n/,contains:[b,{begin:/\[/,end:/\]/, +relevance:0,contains:[b]}]}]},TITLE_MODE:{className:"title",begin:h,relevance:0 +},UNDERSCORE_TITLE_MODE:{className:"title",begin:d,relevance:0},METHOD_GUARD:{ +begin:"\\.\\s*[a-zA-Z_]\\w*",relevance:0},END_SAME_AS_BEGIN:e=>Object.assign(e,{ +"on:begin":(e,t)=>{t.data._beginMatch=e[1]},"on:end":(e,t)=>{ +t.data._beginMatch!==e[1]&&t.ignoreMatch()}})});function k(e,t){ +"."===e.input[e.index-1]&&t.ignoreMatch()}function M(e,t){ +t&&e.beginKeywords&&(e.begin="\\b("+e.beginKeywords.split(" ").join("|")+")(?!\\.)(?=\\b|\\s)", +e.__beforeBegin=k,e.keywords=e.keywords||e.beginKeywords,delete e.beginKeywords, +void 0===e.relevance&&(e.relevance=0))}function O(e,t){ +Array.isArray(e.illegal)&&(e.illegal=((...e)=>"("+e.map((e=>g(e))).join("|")+")")(...e.illegal)) +}function A(e,t){if(e.match){ +if(e.begin||e.end)throw Error("begin & end are not supported with match") +;e.begin=e.match,delete e.match}}function L(e,t){ +void 0===e.relevance&&(e.relevance=1)} +const I=["of","and","for","in","not","or","if","then","parent","list","value"] +;function j(e,t,n="keyword"){const i={} +;return"string"==typeof e?s(n,e.split(" ")):Array.isArray(e)?s(n,e):Object.keys(e).forEach((n=>{ +Object.assign(i,j(e[n],t,n))})),i;function s(e,n){ +t&&(n=n.map((e=>e.toLowerCase()))),n.forEach((t=>{const n=t.split("|") +;i[n[0]]=[e,B(n[0],n[1])]}))}}function B(e,t){ +return t?Number(t):(e=>I.includes(e.toLowerCase()))(e)?0:1} +function T(e,{plugins:t}){function n(t,n){ +return RegExp(g(t),"m"+(e.case_insensitive?"i":"")+(n?"g":""))}class i{ +constructor(){ +this.matchIndexes={},this.regexes=[],this.matchAt=1,this.position=0} +addRule(e,t){ +t.position=this.position++,this.matchIndexes[this.matchAt]=t,this.regexes.push([t,e]), +this.matchAt+=(e=>RegExp(e.toString()+"|").exec("").length-1)(e)+1}compile(){ +0===this.regexes.length&&(this.exec=()=>null) +;const e=this.regexes.map((e=>e[1]));this.matcherRe=n(((e,t="|")=>{let n=0 +;return e.map((e=>{n+=1;const t=n;let i=g(e),s="";for(;i.length>0;){ +const e=u.exec(i);if(!e){s+=i;break} +s+=i.substring(0,e.index),i=i.substring(e.index+e[0].length), +"\\"===e[0][0]&&e[1]?s+="\\"+(Number(e[1])+t):(s+=e[0],"("===e[0]&&n++)}return s +})).map((e=>`(${e})`)).join(t)})(e),!0),this.lastIndex=0}exec(e){ +this.matcherRe.lastIndex=this.lastIndex;const t=this.matcherRe.exec(e) +;if(!t)return null +;const n=t.findIndex(((e,t)=>t>0&&void 0!==e)),i=this.matchIndexes[n] +;return t.splice(0,n),Object.assign(t,i)}}class s{constructor(){ +this.rules=[],this.multiRegexes=[], +this.count=0,this.lastIndex=0,this.regexIndex=0}getMatcher(e){ +if(this.multiRegexes[e])return this.multiRegexes[e];const t=new i +;return this.rules.slice(e).forEach((([e,n])=>t.addRule(e,n))), +t.compile(),this.multiRegexes[e]=t,t}resumingScanAtSamePosition(){ +return 0!==this.regexIndex}considerAll(){this.regexIndex=0}addRule(e,t){ +this.rules.push([e,t]),"begin"===t.type&&this.count++}exec(e){ +const t=this.getMatcher(this.regexIndex);t.lastIndex=this.lastIndex +;let n=t.exec(e) +;if(this.resumingScanAtSamePosition())if(n&&n.index===this.lastIndex);else{ +const t=this.getMatcher(0);t.lastIndex=this.lastIndex+1,n=t.exec(e)} +return n&&(this.regexIndex+=n.position+1, +this.regexIndex===this.count&&this.considerAll()),n}} +if(e.compilerExtensions||(e.compilerExtensions=[]), +e.contains&&e.contains.includes("self"))throw Error("ERR: contains `self` is not supported at the top-level of a language. See documentation.") +;return e.classNameAliases=a(e.classNameAliases||{}),function t(i,r){const l=i +;if(i.isCompiled)return l +;[A].forEach((e=>e(i,r))),e.compilerExtensions.forEach((e=>e(i,r))), +i.__beforeBegin=null,[M,O,L].forEach((e=>e(i,r))),i.isCompiled=!0;let o=null +;if("object"==typeof i.keywords&&(o=i.keywords.$pattern, +delete i.keywords.$pattern), +i.keywords&&(i.keywords=j(i.keywords,e.case_insensitive)), +i.lexemes&&o)throw Error("ERR: Prefer `keywords.$pattern` to `mode.lexemes`, BOTH are not allowed. (see mode reference) ") +;return o=o||i.lexemes||/\w+/, +l.keywordPatternRe=n(o,!0),r&&(i.begin||(i.begin=/\B|\b/), +l.beginRe=n(i.begin),i.endSameAsBegin&&(i.end=i.begin), +i.end||i.endsWithParent||(i.end=/\B|\b/), +i.end&&(l.endRe=n(i.end)),l.terminatorEnd=g(i.end)||"", +i.endsWithParent&&r.terminatorEnd&&(l.terminatorEnd+=(i.end?"|":"")+r.terminatorEnd)), +i.illegal&&(l.illegalRe=n(i.illegal)), +i.contains||(i.contains=[]),i.contains=[].concat(...i.contains.map((e=>(e=>(e.variants&&!e.cachedVariants&&(e.cachedVariants=e.variants.map((t=>a(e,{ +variants:null},t)))),e.cachedVariants?e.cachedVariants:S(e)?a(e,{ +starts:e.starts?a(e.starts):null +}):Object.isFrozen(e)?a(e):e))("self"===e?i:e)))),i.contains.forEach((e=>{t(e,l) +})),i.starts&&t(i.starts,r),l.matcher=(e=>{const t=new s +;return e.contains.forEach((e=>t.addRule(e.begin,{rule:e,type:"begin" +}))),e.terminatorEnd&&t.addRule(e.terminatorEnd,{type:"end" +}),e.illegal&&t.addRule(e.illegal,{type:"illegal"}),t})(l),l}(e)}function S(e){ +return!!e&&(e.endsWithParent||S(e.starts))}function P(e){const t={ +props:["language","code","autodetect"],data:()=>({detectedLanguage:"", +unknownLanguage:!1}),computed:{className(){ +return this.unknownLanguage?"":"hljs "+this.detectedLanguage},highlighted(){ +if(!this.autoDetect&&!e.getLanguage(this.language))return console.warn(`The language "${this.language}" you specified could not be found.`), +this.unknownLanguage=!0,s(this.code);let t={} +;return this.autoDetect?(t=e.highlightAuto(this.code), +this.detectedLanguage=t.language):(t=e.highlight(this.language,this.code,this.ignoreIllegals), +this.detectedLanguage=this.language),t.value},autoDetect(){ +return!(this.language&&(e=this.autodetect,!e&&""!==e));var e}, +ignoreIllegals:()=>!0},render(e){return e("pre",{},[e("code",{ +class:this.className,domProps:{innerHTML:this.highlighted}})])}};return{ +Component:t,VuePlugin:{install(e){e.component("highlightjs",t)}}}}const D={ +"after:highlightElement":({el:e,result:t,text:n})=>{const i=H(e) +;if(!i.length)return;const a=document.createElement("div") +;a.innerHTML=t.value,t.value=((e,t,n)=>{let i=0,a="";const r=[];function l(){ +return e.length&&t.length?e[0].offset!==t[0].offset?e[0].offset"}function c(e){ +a+=""}function g(e){("start"===e.event?o:c)(e.node)} +for(;e.length||t.length;){let t=l() +;if(a+=s(n.substring(i,t[0].offset)),i=t[0].offset,t===e){r.reverse().forEach(c) +;do{g(t.splice(0,1)[0]),t=l()}while(t===e&&t.length&&t[0].offset===i) +;r.reverse().forEach(o) +}else"start"===t[0].event?r.push(t[0].node):r.pop(),g(t.splice(0,1)[0])} +return a+s(n.substr(i))})(i,H(a),n)}};function C(e){ +return e.nodeName.toLowerCase()}function H(e){const t=[];return function e(n,i){ +for(let s=n.firstChild;s;s=s.nextSibling)3===s.nodeType?i+=s.nodeValue.length:1===s.nodeType&&(t.push({ +event:"start",offset:i,node:s}),i=e(s,i),C(s).match(/br|hr|img|input/)||t.push({ +event:"stop",offset:i,node:s}));return i}(e,0),t}const $={},U=e=>{ +console.error(e)},z=(e,...t)=>{console.log("WARN: "+e,...t)},K=(e,t)=>{ +$[`${e}/${t}`]||(console.log(`Deprecated as of ${e}. ${t}`),$[`${e}/${t}`]=!0) +},G=s,V=a,W=Symbol("nomatch");return(e=>{ +const n=Object.create(null),s=Object.create(null),a=[];let r=!0 +;const l=/(^(<[^>]+>|\t|)+|\n)/gm,o="Could not find the language '{}', did you forget to load/include a language module?",g={ +disableAutodetect:!0,name:"Plain text",contains:[]};let u={ +noHighlightRe:/^(no-?highlight)$/i, +languageDetectRe:/\blang(?:uage)?-([\w-]+)\b/i,classPrefix:"hljs-", +tabReplace:null,useBR:!1,languages:null,__emitter:c};function h(e){ +return u.noHighlightRe.test(e)}function d(e,t,n,i){let s="",a="" +;"object"==typeof t?(s=e, +n=t.ignoreIllegals,a=t.language,i=void 0):(K("10.7.0","highlight(lang, code, ...args) has been deprecated."), +K("10.7.0","Please use highlight(code, options) instead.\nhttps://github.com/highlightjs/highlight.js/issues/2277"), +a=e,s=t);const r={code:s,language:a};M("before:highlight",r) +;const l=r.result?r.result:f(r.language,r.code,n,i) +;return l.code=r.code,M("after:highlight",l),l}function f(e,t,s,l){ +function c(e,t){const n=v.case_insensitive?t[0].toLowerCase():t[0] +;return Object.prototype.hasOwnProperty.call(e.keywords,n)&&e.keywords[n]} +function g(){null!=R.subLanguage?(()=>{if(""===M)return;let e=null +;if("string"==typeof R.subLanguage){ +if(!n[R.subLanguage])return void k.addText(M) +;e=f(R.subLanguage,M,!0,_[R.subLanguage]),_[R.subLanguage]=e.top +}else e=p(M,R.subLanguage.length?R.subLanguage:null) +;R.relevance>0&&(O+=e.relevance),k.addSublanguage(e.emitter,e.language) +})():(()=>{if(!R.keywords)return void k.addText(M);let e=0 +;R.keywordPatternRe.lastIndex=0;let t=R.keywordPatternRe.exec(M),n="";for(;t;){ +n+=M.substring(e,t.index);const i=c(R,t);if(i){const[e,s]=i +;if(k.addText(n),n="",O+=s,e.startsWith("_"))n+=t[0];else{ +const n=v.classNameAliases[e]||e;k.addKeyword(t[0],n)}}else n+=t[0] +;e=R.keywordPatternRe.lastIndex,t=R.keywordPatternRe.exec(M)} +n+=M.substr(e),k.addText(n)})(),M=""}function h(e){ +return e.className&&k.openNode(v.classNameAliases[e.className]||e.className), +R=Object.create(e,{parent:{value:R}}),R}function d(e,t,n){let s=((e,t)=>{ +const n=e&&e.exec(t);return n&&0===n.index})(e.endRe,n);if(s){if(e["on:end"]){ +const n=new i(e);e["on:end"](t,n),n.isMatchIgnored&&(s=!1)}if(s){ +for(;e.endsParent&&e.parent;)e=e.parent;return e}} +if(e.endsWithParent)return d(e.parent,t,n)}function m(e){ +return 0===R.matcher.regexIndex?(M+=e[0],1):(I=!0,0)}function b(e){ +const n=e[0],i=t.substr(e.index),s=d(R,e,i);if(!s)return W;const a=R +;a.skip?M+=n:(a.returnEnd||a.excludeEnd||(M+=n),g(),a.excludeEnd&&(M=n));do{ +R.className&&k.closeNode(),R.skip||R.subLanguage||(O+=R.relevance),R=R.parent +}while(R!==s.parent) +;return s.starts&&(s.endSameAsBegin&&(s.starts.endRe=s.endRe), +h(s.starts)),a.returnEnd?0:n.length}let E={};function x(n,a){const l=a&&a[0] +;if(M+=n,null==l)return g(),0 +;if("begin"===E.type&&"end"===a.type&&E.index===a.index&&""===l){ +if(M+=t.slice(a.index,a.index+1),!r){const t=Error("0 width match regex") +;throw t.languageName=e,t.badRule=E.rule,t}return 1} +if(E=a,"begin"===a.type)return function(e){ +const t=e[0],n=e.rule,s=new i(n),a=[n.__beforeBegin,n["on:begin"]] +;for(const n of a)if(n&&(n(e,s),s.isMatchIgnored))return m(t) +;return n&&n.endSameAsBegin&&(n.endRe=RegExp(t.replace(/[-/\\^$*+?.()|[\]{}]/g,"\\$&"),"m")), +n.skip?M+=t:(n.excludeBegin&&(M+=t), +g(),n.returnBegin||n.excludeBegin||(M=t)),h(n),n.returnBegin?0:t.length}(a) +;if("illegal"===a.type&&!s){ +const e=Error('Illegal lexeme "'+l+'" for mode "'+(R.className||"")+'"') +;throw e.mode=R,e}if("end"===a.type){const e=b(a);if(e!==W)return e} +if("illegal"===a.type&&""===l)return 1 +;if(L>1e5&&L>3*a.index)throw Error("potential infinite loop, way more iterations than matches") +;return M+=l,l.length}const v=N(e) +;if(!v)throw U(o.replace("{}",e)),Error('Unknown language: "'+e+'"') +;const w=T(v,{plugins:a});let y="",R=l||w;const _={},k=new u.__emitter(u);(()=>{ +const e=[];for(let t=R;t!==v;t=t.parent)t.className&&e.unshift(t.className) +;e.forEach((e=>k.openNode(e)))})();let M="",O=0,A=0,L=0,I=!1;try{ +for(R.matcher.considerAll();;){ +L++,I?I=!1:R.matcher.considerAll(),R.matcher.lastIndex=A +;const e=R.matcher.exec(t);if(!e)break;const n=x(t.substring(A,e.index),e) +;A=e.index+n}return x(t.substr(A)),k.closeAllNodes(),k.finalize(),y=k.toHTML(),{ +relevance:Math.floor(O),value:y,language:e,illegal:!1,emitter:k,top:R}}catch(n){ +if(n.message&&n.message.includes("Illegal"))return{illegal:!0,illegalBy:{ +msg:n.message,context:t.slice(A-100,A+100),mode:n.mode},sofar:y,relevance:0, +value:G(t),emitter:k};if(r)return{illegal:!1,relevance:0,value:G(t),emitter:k, +language:e,top:R,errorRaised:n};throw n}}function p(e,t){ +t=t||u.languages||Object.keys(n);const i=(e=>{const t={relevance:0, +emitter:new u.__emitter(u),value:G(e),illegal:!1,top:g} +;return t.emitter.addText(e),t})(e),s=t.filter(N).filter(k).map((t=>f(t,e,!1))) +;s.unshift(i);const a=s.sort(((e,t)=>{ +if(e.relevance!==t.relevance)return t.relevance-e.relevance +;if(e.language&&t.language){if(N(e.language).supersetOf===t.language)return 1 +;if(N(t.language).supersetOf===e.language)return-1}return 0})),[r,l]=a,o=r +;return o.second_best=l,o}const m={"before:highlightElement":({el:e})=>{ +u.useBR&&(e.innerHTML=e.innerHTML.replace(/\n/g,"").replace(//g,"\n")) +},"after:highlightElement":({result:e})=>{ +u.useBR&&(e.value=e.value.replace(/\n/g,"
"))}},b=/^(<[^>]+>|\t)+/gm,E={ +"after:highlightElement":({result:e})=>{ +u.tabReplace&&(e.value=e.value.replace(b,(e=>e.replace(/\t/g,u.tabReplace))))}} +;function x(e){let t=null;const n=(e=>{let t=e.className+" " +;t+=e.parentNode?e.parentNode.className:"";const n=u.languageDetectRe.exec(t) +;if(n){const t=N(n[1]) +;return t||(z(o.replace("{}",n[1])),z("Falling back to no-highlight mode for this block.",e)), +t?n[1]:"no-highlight"}return t.split(/\s+/).find((e=>h(e)||N(e)))})(e) +;if(h(n))return;M("before:highlightElement",{el:e,language:n}),t=e +;const i=t.textContent,a=n?d(i,{language:n,ignoreIllegals:!0}):p(i) +;M("after:highlightElement",{el:e,result:a,text:i +}),e.innerHTML=a.value,((e,t,n)=>{const i=t?s[t]:n +;e.classList.add("hljs"),i&&e.classList.add(i)})(e,n,a.language),e.result={ +language:a.language,re:a.relevance,relavance:a.relevance +},a.second_best&&(e.second_best={language:a.second_best.language, +re:a.second_best.relevance,relavance:a.second_best.relevance})}const v=()=>{ +v.called||(v.called=!0, +K("10.6.0","initHighlighting() is deprecated. Use highlightAll() instead."), +document.querySelectorAll("pre code").forEach(x))};let w=!1;function y(){ +"loading"!==document.readyState?document.querySelectorAll("pre code").forEach(x):w=!0 +}function N(e){return e=(e||"").toLowerCase(),n[e]||n[s[e]]} +function R(e,{languageName:t}){"string"==typeof e&&(e=[e]),e.forEach((e=>{ +s[e.toLowerCase()]=t}))}function k(e){const t=N(e) +;return t&&!t.disableAutodetect}function M(e,t){const n=e;a.forEach((e=>{ +e[n]&&e[n](t)}))} +"undefined"!=typeof window&&window.addEventListener&&window.addEventListener("DOMContentLoaded",(()=>{ +w&&y()}),!1),Object.assign(e,{highlight:d,highlightAuto:p,highlightAll:y, +fixMarkup:e=>{ +return K("10.2.0","fixMarkup will be removed entirely in v11.0"),K("10.2.0","Please see https://github.com/highlightjs/highlight.js/issues/2534"), +t=e, +u.tabReplace||u.useBR?t.replace(l,(e=>"\n"===e?u.useBR?"
":e:u.tabReplace?e.replace(/\t/g,u.tabReplace):e)):t +;var t},highlightElement:x, +highlightBlock:e=>(K("10.7.0","highlightBlock will be removed entirely in v12.0"), +K("10.7.0","Please use highlightElement now."),x(e)),configure:e=>{ +e.useBR&&(K("10.3.0","'useBR' will be removed entirely in v11.0"), +K("10.3.0","Please see https://github.com/highlightjs/highlight.js/issues/2559")), +u=V(u,e)},initHighlighting:v,initHighlightingOnLoad:()=>{ +K("10.6.0","initHighlightingOnLoad() is deprecated. Use highlightAll() instead."), +w=!0},registerLanguage:(t,i)=>{let s=null;try{s=i(e)}catch(e){ +if(U("Language definition for '{}' could not be registered.".replace("{}",t)), +!r)throw e;U(e),s=g} +s.name||(s.name=t),n[t]=s,s.rawDefinition=i.bind(null,e),s.aliases&&R(s.aliases,{ +languageName:t})},unregisterLanguage:e=>{delete n[e] +;for(const t of Object.keys(s))s[t]===e&&delete s[t]}, +listLanguages:()=>Object.keys(n),getLanguage:N,registerAliases:R, +requireLanguage:e=>{ +K("10.4.0","requireLanguage will be removed entirely in v11."), +K("10.4.0","Please see https://github.com/highlightjs/highlight.js/pull/2844") +;const t=N(e);if(t)return t +;throw Error("The '{}' language is required, but not loaded.".replace("{}",e))}, +autoDetection:k,inherit:V,addPlugin:e=>{(e=>{ +e["before:highlightBlock"]&&!e["before:highlightElement"]&&(e["before:highlightElement"]=t=>{ +e["before:highlightBlock"](Object.assign({block:t.el},t)) +}),e["after:highlightBlock"]&&!e["after:highlightElement"]&&(e["after:highlightElement"]=t=>{ +e["after:highlightBlock"](Object.assign({block:t.el},t))})})(e),a.push(e)}, +vuePlugin:P(e).VuePlugin}),e.debugMode=()=>{r=!1},e.safeMode=()=>{r=!0 +},e.versionString="10.7.2";for(const e in _)"object"==typeof _[e]&&t(_[e]) +;return Object.assign(e,_),e.addPlugin(m),e.addPlugin(D),e.addPlugin(E),e})({}) +}();"object"==typeof exports&&"undefined"!=typeof module&&(module.exports=hljs);hljs.registerLanguage("sql",(()=>{"use strict";function e(e){ +return e?"string"==typeof e?e:e.source:null}function r(...r){ +return r.map((r=>e(r))).join("")}function t(...r){ +return"("+r.map((r=>e(r))).join("|")+")"}return e=>{ +const n=e.COMMENT("--","$"),a=["true","false","unknown"],i=["bigint","binary","blob","boolean","char","character","clob","date","dec","decfloat","decimal","float","int","integer","interval","nchar","nclob","national","numeric","real","row","smallint","time","timestamp","varchar","varying","varbinary"],s=["abs","acos","array_agg","asin","atan","avg","cast","ceil","ceiling","coalesce","corr","cos","cosh","count","covar_pop","covar_samp","cume_dist","dense_rank","deref","element","exp","extract","first_value","floor","json_array","json_arrayagg","json_exists","json_object","json_objectagg","json_query","json_table","json_table_primitive","json_value","lag","last_value","lead","listagg","ln","log","log10","lower","max","min","mod","nth_value","ntile","nullif","percent_rank","percentile_cont","percentile_disc","position","position_regex","power","rank","regr_avgx","regr_avgy","regr_count","regr_intercept","regr_r2","regr_slope","regr_sxx","regr_sxy","regr_syy","row_number","sin","sinh","sqrt","stddev_pop","stddev_samp","substring","substring_regex","sum","tan","tanh","translate","translate_regex","treat","trim","trim_array","unnest","upper","value_of","var_pop","var_samp","width_bucket"],o=["create table","insert into","primary key","foreign key","not null","alter table","add constraint","grouping sets","on overflow","character set","respect nulls","ignore nulls","nulls first","nulls last","depth first","breadth first"],c=s,l=["abs","acos","all","allocate","alter","and","any","are","array","array_agg","array_max_cardinality","as","asensitive","asin","asymmetric","at","atan","atomic","authorization","avg","begin","begin_frame","begin_partition","between","bigint","binary","blob","boolean","both","by","call","called","cardinality","cascaded","case","cast","ceil","ceiling","char","char_length","character","character_length","check","classifier","clob","close","coalesce","collate","collect","column","commit","condition","connect","constraint","contains","convert","copy","corr","corresponding","cos","cosh","count","covar_pop","covar_samp","create","cross","cube","cume_dist","current","current_catalog","current_date","current_default_transform_group","current_path","current_role","current_row","current_schema","current_time","current_timestamp","current_path","current_role","current_transform_group_for_type","current_user","cursor","cycle","date","day","deallocate","dec","decimal","decfloat","declare","default","define","delete","dense_rank","deref","describe","deterministic","disconnect","distinct","double","drop","dynamic","each","element","else","empty","end","end_frame","end_partition","end-exec","equals","escape","every","except","exec","execute","exists","exp","external","extract","false","fetch","filter","first_value","float","floor","for","foreign","frame_row","free","from","full","function","fusion","get","global","grant","group","grouping","groups","having","hold","hour","identity","in","indicator","initial","inner","inout","insensitive","insert","int","integer","intersect","intersection","interval","into","is","join","json_array","json_arrayagg","json_exists","json_object","json_objectagg","json_query","json_table","json_table_primitive","json_value","lag","language","large","last_value","lateral","lead","leading","left","like","like_regex","listagg","ln","local","localtime","localtimestamp","log","log10","lower","match","match_number","match_recognize","matches","max","member","merge","method","min","minute","mod","modifies","module","month","multiset","national","natural","nchar","nclob","new","no","none","normalize","not","nth_value","ntile","null","nullif","numeric","octet_length","occurrences_regex","of","offset","old","omit","on","one","only","open","or","order","out","outer","over","overlaps","overlay","parameter","partition","pattern","per","percent","percent_rank","percentile_cont","percentile_disc","period","portion","position","position_regex","power","precedes","precision","prepare","primary","procedure","ptf","range","rank","reads","real","recursive","ref","references","referencing","regr_avgx","regr_avgy","regr_count","regr_intercept","regr_r2","regr_slope","regr_sxx","regr_sxy","regr_syy","release","result","return","returns","revoke","right","rollback","rollup","row","row_number","rows","running","savepoint","scope","scroll","search","second","seek","select","sensitive","session_user","set","show","similar","sin","sinh","skip","smallint","some","specific","specifictype","sql","sqlexception","sqlstate","sqlwarning","sqrt","start","static","stddev_pop","stddev_samp","submultiset","subset","substring","substring_regex","succeeds","sum","symmetric","system","system_time","system_user","table","tablesample","tan","tanh","then","time","timestamp","timezone_hour","timezone_minute","to","trailing","translate","translate_regex","translation","treat","trigger","trim","trim_array","true","truncate","uescape","union","unique","unknown","unnest","update ","upper","user","using","value","values","value_of","var_pop","var_samp","varbinary","varchar","varying","versioning","when","whenever","where","width_bucket","window","with","within","without","year","add","asc","collation","desc","final","first","last","view"].filter((e=>!s.includes(e))),u={ +begin:r(/\b/,t(...c),/\s*\(/),keywords:{built_in:c}};return{name:"SQL", +case_insensitive:!0,illegal:/[{}]|<\//,keywords:{$pattern:/\b[\w\.]+/, +keyword:((e,{exceptions:r,when:t}={})=>{const n=t +;return r=r||[],e.map((e=>e.match(/\|\d+$/)||r.includes(e)?e:n(e)?e+"|0":e)) +})(l,{when:e=>e.length<3}),literal:a,type:i, +built_in:["current_catalog","current_date","current_default_transform_group","current_path","current_role","current_schema","current_transform_group_for_type","current_user","session_user","system_time","system_user","current_time","localtime","current_timestamp","localtimestamp"] +},contains:[{begin:t(...o),keywords:{$pattern:/[\w\.]+/,keyword:l.concat(o), +literal:a,type:i}},{className:"type", +begin:t("double precision","large object","with timezone","without timezone") +},u,{className:"variable",begin:/@[a-z0-9]+/},{className:"string",variants:[{ +begin:/'/,end:/'/,contains:[{begin:/''/}]}]},{begin:/"/,end:/"/,contains:[{ +begin:/""/}]},e.C_NUMBER_MODE,e.C_BLOCK_COMMENT_MODE,n,{className:"operator", +begin:/[-+*/=%^~]|&&?|\|\|?|!=?|<(?:=>?|<|>)?|>[>=]?/,relevance:0}]}}})());hljs.registerLanguage("ini",(()=>{"use strict";function e(e){ +return e?"string"==typeof e?e:e.source:null}function n(...n){ +return n.map((n=>e(n))).join("")}return s=>{const a={className:"number", +relevance:0,variants:[{begin:/([+-]+)?[\d]+_[\d_]+/},{begin:s.NUMBER_RE}] +},i=s.COMMENT();i.variants=[{begin:/;/,end:/$/},{begin:/#/,end:/$/}];const t={ +className:"variable",variants:[{begin:/\$[\w\d"][\w\d_]*/},{begin:/\$\{(.*?)\}/ +}]},r={className:"literal",begin:/\bon|off|true|false|yes|no\b/},l={ +className:"string",contains:[s.BACKSLASH_ESCAPE],variants:[{begin:"'''", +end:"'''",relevance:10},{begin:'"""',end:'"""',relevance:10},{begin:'"',end:'"' +},{begin:"'",end:"'"}]},c={begin:/\[/,end:/\]/,contains:[i,r,t,l,a,"self"], +relevance:0 +},g="("+[/[A-Za-z0-9_-]+/,/"(\\"|[^"])*"/,/'[^']*'/].map((n=>e(n))).join("|")+")" +;return{name:"TOML, also INI",aliases:["toml"],case_insensitive:!0,illegal:/\S/, +contains:[i,{className:"section",begin:/\[+/,end:/\]+/},{ +begin:n(g,"(\\s*\\.\\s*",g,")*",n("(?=",/\s*=\s*[^#\s]/,")")),className:"attr", +starts:{end:/$/,contains:[i,c,r,t,l,a]}}]}}})());hljs.registerLanguage("csharp",(()=>{"use strict";return e=>{const n={ +keyword:["abstract","as","base","break","case","class","const","continue","do","else","event","explicit","extern","finally","fixed","for","foreach","goto","if","implicit","in","interface","internal","is","lock","namespace","new","operator","out","override","params","private","protected","public","readonly","record","ref","return","sealed","sizeof","stackalloc","static","struct","switch","this","throw","try","typeof","unchecked","unsafe","using","virtual","void","volatile","while"].concat(["add","alias","and","ascending","async","await","by","descending","equals","from","get","global","group","init","into","join","let","nameof","not","notnull","on","or","orderby","partial","remove","select","set","unmanaged","value|0","var","when","where","with","yield"]), +built_in:["bool","byte","char","decimal","delegate","double","dynamic","enum","float","int","long","nint","nuint","object","sbyte","short","string","ulong","uint","ushort"], +literal:["default","false","null","true"]},a=e.inherit(e.TITLE_MODE,{ +begin:"[a-zA-Z](\\.?\\w)*"}),i={className:"number",variants:[{ +begin:"\\b(0b[01']+)"},{ +begin:"(-?)\\b([\\d']+(\\.[\\d']*)?|\\.[\\d']+)(u|U|l|L|ul|UL|f|F|b|B)"},{ +begin:"(-?)(\\b0[xX][a-fA-F0-9']+|(\\b[\\d']+(\\.[\\d']*)?|\\.[\\d']+)([eE][-+]?[\\d']+)?)" +}],relevance:0},s={className:"string",begin:'@"',end:'"',contains:[{begin:'""'}] +},t=e.inherit(s,{illegal:/\n/}),r={className:"subst",begin:/\{/,end:/\}/, +keywords:n},l=e.inherit(r,{illegal:/\n/}),c={className:"string",begin:/\$"/, +end:'"',illegal:/\n/,contains:[{begin:/\{\{/},{begin:/\}\}/ +},e.BACKSLASH_ESCAPE,l]},o={className:"string",begin:/\$@"/,end:'"',contains:[{ +begin:/\{\{/},{begin:/\}\}/},{begin:'""'},r]},d=e.inherit(o,{illegal:/\n/, +contains:[{begin:/\{\{/},{begin:/\}\}/},{begin:'""'},l]}) +;r.contains=[o,c,s,e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,i,e.C_BLOCK_COMMENT_MODE], +l.contains=[d,c,t,e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,i,e.inherit(e.C_BLOCK_COMMENT_MODE,{ +illegal:/\n/})];const g={variants:[o,c,s,e.APOS_STRING_MODE,e.QUOTE_STRING_MODE] +},E={begin:"<",end:">",contains:[{beginKeywords:"in out"},a] +},_=e.IDENT_RE+"(<"+e.IDENT_RE+"(\\s*,\\s*"+e.IDENT_RE+")*>)?(\\[\\])?",b={ +begin:"@"+e.IDENT_RE,relevance:0};return{name:"C#",aliases:["cs","c#"], +keywords:n,illegal:/::/,contains:[e.COMMENT("///","$",{returnBegin:!0, +contains:[{className:"doctag",variants:[{begin:"///",relevance:0},{ +begin:"\x3c!--|--\x3e"},{begin:""}]}] +}),e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,{className:"meta",begin:"#", +end:"$",keywords:{ +"meta-keyword":"if else elif endif define undef warning error line region endregion pragma checksum" +}},g,i,{beginKeywords:"class interface",relevance:0,end:/[{;=]/, +illegal:/[^\s:,]/,contains:[{beginKeywords:"where class" +},a,E,e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]},{beginKeywords:"namespace", +relevance:0,end:/[{;=]/,illegal:/[^\s:]/, +contains:[a,e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]},{ +beginKeywords:"record",relevance:0,end:/[{;=]/,illegal:/[^\s:]/, +contains:[a,E,e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]},{className:"meta", +begin:"^\\s*\\[",excludeBegin:!0,end:"\\]",excludeEnd:!0,contains:[{ +className:"meta-string",begin:/"/,end:/"/}]},{ +beginKeywords:"new return throw await else",relevance:0},{className:"function", +begin:"("+_+"\\s+)+"+e.IDENT_RE+"\\s*(<.+>\\s*)?\\(",returnBegin:!0, +end:/\s*[{;=]/,excludeEnd:!0,keywords:n,contains:[{ +beginKeywords:"public private protected static internal protected abstract async extern override unsafe virtual new sealed partial", +relevance:0},{begin:e.IDENT_RE+"\\s*(<.+>\\s*)?\\(",returnBegin:!0, +contains:[e.TITLE_MODE,E],relevance:0},{className:"params",begin:/\(/,end:/\)/, +excludeBegin:!0,excludeEnd:!0,keywords:n,relevance:0, +contains:[g,i,e.C_BLOCK_COMMENT_MODE] +},e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]},b]}}})());hljs.registerLanguage("perl",(()=>{"use strict";function e(e){ +return e?"string"==typeof e?e:e.source:null}function n(...n){ +return n.map((n=>e(n))).join("")}function t(...n){ +return"("+n.map((n=>e(n))).join("|")+")"}return e=>{ +const r=/[dualxmsipngr]{0,12}/,s={$pattern:/[\w.]+/, +keyword:"abs accept alarm and atan2 bind binmode bless break caller chdir chmod chomp chop chown chr chroot close closedir connect continue cos crypt dbmclose dbmopen defined delete die do dump each else elsif endgrent endhostent endnetent endprotoent endpwent endservent eof eval exec exists exit exp fcntl fileno flock for foreach fork format formline getc getgrent getgrgid getgrnam gethostbyaddr gethostbyname gethostent getlogin getnetbyaddr getnetbyname getnetent getpeername getpgrp getpriority getprotobyname getprotobynumber getprotoent getpwent getpwnam getpwuid getservbyname getservbyport getservent getsockname getsockopt given glob gmtime goto grep gt hex if index int ioctl join keys kill last lc lcfirst length link listen local localtime log lstat lt ma map mkdir msgctl msgget msgrcv msgsnd my ne next no not oct open opendir or ord our pack package pipe pop pos print printf prototype push q|0 qq quotemeta qw qx rand read readdir readline readlink readpipe recv redo ref rename require reset return reverse rewinddir rindex rmdir say scalar seek seekdir select semctl semget semop send setgrent sethostent setnetent setpgrp setpriority setprotoent setpwent setservent setsockopt shift shmctl shmget shmread shmwrite shutdown sin sleep socket socketpair sort splice split sprintf sqrt srand stat state study sub substr symlink syscall sysopen sysread sysseek system syswrite tell telldir tie tied time times tr truncate uc ucfirst umask undef unless unlink unpack unshift untie until use utime values vec wait waitpid wantarray warn when while write x|0 xor y|0" +},i={className:"subst",begin:"[$@]\\{",end:"\\}",keywords:s},a={begin:/->\{/, +end:/\}/},o={variants:[{begin:/\$\d/},{ +begin:n(/[$%@](\^\w\b|#\w+(::\w+)*|\{\w+\}|\w+(::\w*)*)/,"(?![A-Za-z])(?![@$%])") +},{begin:/[$%@][^\s\w{]/,relevance:0}] +},c=[e.BACKSLASH_ESCAPE,i,o],g=[/!/,/\//,/\|/,/\?/,/'/,/"/,/#/],l=(e,t,s="\\1")=>{ +const i="\\1"===s?s:n(s,t) +;return n(n("(?:",e,")"),t,/(?:\\.|[^\\\/])*?/,i,/(?:\\.|[^\\\/])*?/,s,r) +},d=(e,t,s)=>n(n("(?:",e,")"),t,/(?:\\.|[^\\\/])*?/,s,r),p=[o,e.HASH_COMMENT_MODE,e.COMMENT(/^=\w/,/=cut/,{ +endsWithParent:!0}),a,{className:"string",contains:c,variants:[{ +begin:"q[qwxr]?\\s*\\(",end:"\\)",relevance:5},{begin:"q[qwxr]?\\s*\\[", +end:"\\]",relevance:5},{begin:"q[qwxr]?\\s*\\{",end:"\\}",relevance:5},{ +begin:"q[qwxr]?\\s*\\|",end:"\\|",relevance:5},{begin:"q[qwxr]?\\s*<",end:">", +relevance:5},{begin:"qw\\s+q",end:"q",relevance:5},{begin:"'",end:"'", +contains:[e.BACKSLASH_ESCAPE]},{begin:'"',end:'"'},{begin:"`",end:"`", +contains:[e.BACKSLASH_ESCAPE]},{begin:/\{\w+\}/,relevance:0},{ +begin:"-?\\w+\\s*=>",relevance:0}]},{className:"number", +begin:"(\\b0[0-7_]+)|(\\b0x[0-9a-fA-F_]+)|(\\b[1-9][0-9_]*(\\.[0-9_]+)?)|[0_]\\b", +relevance:0},{ +begin:"(\\/\\/|"+e.RE_STARTERS_RE+"|\\b(split|return|print|reverse|grep)\\b)\\s*", +keywords:"split return print reverse grep",relevance:0, +contains:[e.HASH_COMMENT_MODE,{className:"regexp",variants:[{ +begin:l("s|tr|y",t(...g))},{begin:l("s|tr|y","\\(","\\)")},{ +begin:l("s|tr|y","\\[","\\]")},{begin:l("s|tr|y","\\{","\\}")}],relevance:2},{ +className:"regexp",variants:[{begin:/(m|qr)\/\//,relevance:0},{ +begin:d("(?:m|qr)?",/\//,/\//)},{begin:d("m|qr",t(...g),/\1/)},{ +begin:d("m|qr",/\(/,/\)/)},{begin:d("m|qr",/\[/,/\]/)},{ +begin:d("m|qr",/\{/,/\}/)}]}]},{className:"function",beginKeywords:"sub", +end:"(\\s*\\(.*?\\))?[;{]",excludeEnd:!0,relevance:5,contains:[e.TITLE_MODE]},{ +begin:"-\\w\\b",relevance:0},{begin:"^__DATA__$",end:"^__END__$", +subLanguage:"mojolicious",contains:[{begin:"^@@.*",end:"$",className:"comment"}] +}];return i.contains=p,a.contains=p,{name:"Perl",aliases:["pl","pm"],keywords:s, +contains:p}}})());hljs.registerLanguage("typescript",(()=>{"use strict" +;const e="[A-Za-z$_][0-9A-Za-z$_]*",n=["as","in","of","if","for","while","finally","var","new","function","do","return","void","else","break","catch","instanceof","with","throw","case","default","try","switch","continue","typeof","delete","let","yield","const","class","debugger","async","await","static","import","from","export","extends"],a=["true","false","null","undefined","NaN","Infinity"],s=[].concat(["setInterval","setTimeout","clearInterval","clearTimeout","require","exports","eval","isFinite","isNaN","parseFloat","parseInt","decodeURI","decodeURIComponent","encodeURI","encodeURIComponent","escape","unescape"],["arguments","this","super","console","window","document","localStorage","module","global"],["Intl","DataView","Number","Math","Date","String","RegExp","Object","Function","Boolean","Error","Symbol","Set","Map","WeakSet","WeakMap","Proxy","Reflect","JSON","Promise","Float64Array","Int16Array","Int32Array","Int8Array","Uint16Array","Uint32Array","Float32Array","Array","Uint8Array","Uint8ClampedArray","ArrayBuffer","BigInt64Array","BigUint64Array","BigInt"],["EvalError","InternalError","RangeError","ReferenceError","SyntaxError","TypeError","URIError"]) +;function t(e){return r("(?=",e,")")}function r(...e){return e.map((e=>{ +return(n=e)?"string"==typeof n?n:n.source:null;var n})).join("")}return i=>{ +const c={$pattern:e, +keyword:n.concat(["type","namespace","typedef","interface","public","private","protected","implements","declare","abstract","readonly"]), +literal:a, +built_in:s.concat(["any","void","number","boolean","string","object","never","enum"]) +},o={className:"meta",begin:"@[A-Za-z$_][0-9A-Za-z$_]*"},l=(e,n,a)=>{ +const s=e.contains.findIndex((e=>e.label===n)) +;if(-1===s)throw Error("can not find mode to replace");e.contains.splice(s,1,a) +},b=(i=>{const c=e,o={begin:/<[A-Za-z0-9\\._:-]+/, +end:/\/[A-Za-z0-9\\._:-]+>|\/>/,isTrulyOpeningTag:(e,n)=>{ +const a=e[0].length+e.index,s=e.input[a];"<"!==s?">"===s&&(((e,{after:n})=>{ +const a="", +returnBegin:!0,end:"\\s*=>",contains:[{className:"params",variants:[{ +begin:i.UNDERSCORE_IDENT_RE,relevance:0},{className:null,begin:/\(\s*\)/,skip:!0 +},{begin:/\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0,keywords:l,contains:f}]}] +},{begin:/,/,relevance:0},{className:"",begin:/\s/,end:/\s*/,skip:!0},{ +variants:[{begin:"<>",end:""},{begin:o.begin,"on:begin":o.isTrulyOpeningTag, +end:o.end}],subLanguage:"xml",contains:[{begin:o.begin,end:o.end,skip:!0, +contains:["self"]}]}],relevance:0},{className:"function", +beginKeywords:"function",end:/[{;]/,excludeEnd:!0,keywords:l, +contains:["self",i.inherit(i.TITLE_MODE,{begin:c}),A],illegal:/%/},{ +beginKeywords:"while if switch catch for"},{className:"function", +begin:i.UNDERSCORE_IDENT_RE+"\\([^()]*(\\([^()]*(\\([^()]*\\)[^()]*)*\\)[^()]*)*\\)\\s*\\{", +returnBegin:!0,contains:[A,i.inherit(i.TITLE_MODE,{begin:c})]},{variants:[{ +begin:"\\."+c},{begin:"\\$"+c}],relevance:0},{className:"class", +beginKeywords:"class",end:/[{;=]/,excludeEnd:!0,illegal:/[:"[\]]/,contains:[{ +beginKeywords:"extends"},i.UNDERSCORE_TITLE_MODE]},{begin:/\b(?=constructor)/, +end:/[{;]/,excludeEnd:!0,contains:[i.inherit(i.TITLE_MODE,{begin:c}),"self",A] +},{begin:"(get|set)\\s+(?="+c+"\\()",end:/\{/,keywords:"get set", +contains:[i.inherit(i.TITLE_MODE,{begin:c}),{begin:/\(\)/},A]},{begin:/\$[(.]/}] +}})(i) +;return Object.assign(b.keywords,c),b.exports.PARAMS_CONTAINS.push(o),b.contains=b.contains.concat([o,{ +beginKeywords:"namespace",end:/\{/,excludeEnd:!0},{beginKeywords:"interface", +end:/\{/,excludeEnd:!0,keywords:"interface extends" +}]),l(b,"shebang",i.SHEBANG()),l(b,"use_strict",{className:"meta",relevance:10, +begin:/^\s*['"]use strict['"]/ +}),b.contains.find((e=>"function"===e.className)).relevance=0,Object.assign(b,{ +name:"TypeScript",aliases:["ts","tsx"]}),b}})());hljs.registerLanguage("kotlin",(()=>{"use strict" +;var e="\\.([0-9](_*[0-9])*)",n="[0-9a-fA-F](_*[0-9a-fA-F])*",a={ +className:"number",variants:[{ +begin:`(\\b([0-9](_*[0-9])*)((${e})|\\.)?|(${e}))[eE][+-]?([0-9](_*[0-9])*)[fFdD]?\\b` +},{begin:`\\b([0-9](_*[0-9])*)((${e})[fFdD]?\\b|\\.([fFdD]\\b)?)`},{ +begin:`(${e})[fFdD]?\\b`},{begin:"\\b([0-9](_*[0-9])*)[fFdD]\\b"},{ +begin:`\\b0[xX]((${n})\\.?|(${n})?\\.(${n}))[pP][+-]?([0-9](_*[0-9])*)[fFdD]?\\b` +},{begin:"\\b(0|[1-9](_*[0-9])*)[lL]?\\b"},{begin:`\\b0[xX](${n})[lL]?\\b`},{ +begin:"\\b0(_*[0-7])*[lL]?\\b"},{begin:"\\b0[bB][01](_*[01])*[lL]?\\b"}], +relevance:0};return e=>{const n={ +keyword:"abstract as val var vararg get set class object open private protected public noinline crossinline dynamic final enum if else do while for when throw try catch finally import package is in fun override companion reified inline lateinit init interface annotation data sealed internal infix operator out by constructor super tailrec where const inner suspend typealias external expect actual", +built_in:"Byte Short Char Int Long Boolean Float Double Void Unit Nothing", +literal:"true false null"},i={className:"symbol",begin:e.UNDERSCORE_IDENT_RE+"@" +},s={className:"subst",begin:/\$\{/,end:/\}/,contains:[e.C_NUMBER_MODE]},t={ +className:"variable",begin:"\\$"+e.UNDERSCORE_IDENT_RE},r={className:"string", +variants:[{begin:'"""',end:'"""(?=[^"])',contains:[t,s]},{begin:"'",end:"'", +illegal:/\n/,contains:[e.BACKSLASH_ESCAPE]},{begin:'"',end:'"',illegal:/\n/, +contains:[e.BACKSLASH_ESCAPE,t,s]}]};s.contains.push(r);const l={ +className:"meta", +begin:"@(?:file|property|field|get|set|receiver|param|setparam|delegate)\\s*:(?:\\s*"+e.UNDERSCORE_IDENT_RE+")?" +},c={className:"meta",begin:"@"+e.UNDERSCORE_IDENT_RE,contains:[{begin:/\(/, +end:/\)/,contains:[e.inherit(r,{className:"meta-string"})]}] +},o=a,b=e.COMMENT("/\\*","\\*/",{contains:[e.C_BLOCK_COMMENT_MODE]}),E={ +variants:[{className:"type",begin:e.UNDERSCORE_IDENT_RE},{begin:/\(/,end:/\)/, +contains:[]}]},d=E;return d.variants[1].contains=[E],E.variants[1].contains=[d], +{name:"Kotlin",aliases:["kt","kts"],keywords:n, +contains:[e.COMMENT("/\\*\\*","\\*/",{relevance:0,contains:[{className:"doctag", +begin:"@[A-Za-z]+"}]}),e.C_LINE_COMMENT_MODE,b,{className:"keyword", +begin:/\b(break|continue|return|this)\b/,starts:{contains:[{className:"symbol", +begin:/@\w+/}]}},i,l,c,{className:"function",beginKeywords:"fun",end:"[(]|$", +returnBegin:!0,excludeEnd:!0,keywords:n,relevance:5,contains:[{ +begin:e.UNDERSCORE_IDENT_RE+"\\s*\\(",returnBegin:!0,relevance:0, +contains:[e.UNDERSCORE_TITLE_MODE]},{className:"type",begin://, +keywords:"reified",relevance:0},{className:"params",begin:/\(/,end:/\)/, +endsParent:!0,keywords:n,relevance:0,contains:[{begin:/:/,end:/[=,\/]/, +endsWithParent:!0,contains:[E,e.C_LINE_COMMENT_MODE,b],relevance:0 +},e.C_LINE_COMMENT_MODE,b,l,c,r,e.C_NUMBER_MODE]},b]},{className:"class", +beginKeywords:"class interface trait",end:/[:\{(]|$/,excludeEnd:!0, +illegal:"extends implements",contains:[{ +beginKeywords:"public protected internal private constructor" +},e.UNDERSCORE_TITLE_MODE,{className:"type",begin://,excludeBegin:!0, +excludeEnd:!0,relevance:0},{className:"type",begin:/[,:]\s*/,end:/[<\(,]|$/, +excludeBegin:!0,returnEnd:!0},l,c]},r,{className:"meta",begin:"^#!/usr/bin/env", +end:"$",illegal:"\n"},o]}}})());hljs.registerLanguage("xml",(()=>{"use strict";function e(e){ +return e?"string"==typeof e?e:e.source:null}function n(e){return a("(?=",e,")")} +function a(...n){return n.map((n=>e(n))).join("")}function s(...n){ +return"("+n.map((n=>e(n))).join("|")+")"}return e=>{ +const t=a(/[A-Z_]/,a("(",/[A-Z0-9_.-]*:/,")?"),/[A-Z0-9_.-]*/),i={ +className:"symbol",begin:/&[a-z]+;|&#[0-9]+;|&#x[a-f0-9]+;/},r={begin:/\s/, +contains:[{className:"meta-keyword",begin:/#?[a-z_][a-z1-9_-]+/,illegal:/\n/}] +},c=e.inherit(r,{begin:/\(/,end:/\)/}),l=e.inherit(e.APOS_STRING_MODE,{ +className:"meta-string"}),g=e.inherit(e.QUOTE_STRING_MODE,{ +className:"meta-string"}),m={endsWithParent:!0,illegal:/`]+/}]}] +}]};return{name:"HTML, XML", +aliases:["html","xhtml","rss","atom","xjb","xsd","xsl","plist","wsf","svg"], +case_insensitive:!0,contains:[{className:"meta",begin://, +relevance:10,contains:[r,g,l,c,{begin:/\[/,end:/\]/,contains:[{className:"meta", +begin://,contains:[r,c,g,l]}]}]},e.COMMENT(//,{ +relevance:10}),{begin://,relevance:10},i,{ +className:"meta",begin:/<\?xml/,end:/\?>/,relevance:10},{className:"tag", +begin:/)/,end:/>/,keywords:{name:"style"},contains:[m],starts:{ +end:/<\/style>/,returnEnd:!0,subLanguage:["css","xml"]}},{className:"tag", +begin:/)/,end:/>/,keywords:{name:"script"},contains:[m],starts:{ +end:/<\/script>/,returnEnd:!0,subLanguage:["javascript","handlebars","xml"]}},{ +className:"tag",begin:/<>|<\/>/},{className:"tag", +begin:a(//,/>/,/\s/)))),end:/\/?>/,contains:[{className:"name", +begin:t,relevance:0,starts:m}]},{className:"tag",begin:a(/<\//,n(a(t,/>/))), +contains:[{className:"name",begin:t,relevance:0},{begin:/>/,relevance:0, +endsParent:!0}]}]}}})());hljs.registerLanguage("markdown",(()=>{"use strict";function n(...n){ +return n.map((n=>{return(e=n)?"string"==typeof e?e:e.source:null;var e +})).join("")}return e=>{const a={begin:/<\/?[A-Za-z_]/,end:">", +subLanguage:"xml",relevance:0},i={variants:[{begin:/\[.+?\]\[.*?\]/,relevance:0 +},{begin:/\[.+?\]\(((data|javascript|mailto):|(?:http|ftp)s?:\/\/).*?\)/, +relevance:2},{begin:n(/\[.+?\]\(/,/[A-Za-z][A-Za-z0-9+.-]*/,/:\/\/.*?\)/), +relevance:2},{begin:/\[.+?\]\([./?&#].*?\)/,relevance:1},{ +begin:/\[.+?\]\(.*?\)/,relevance:0}],returnBegin:!0,contains:[{ +className:"string",relevance:0,begin:"\\[",end:"\\]",excludeBegin:!0, +returnEnd:!0},{className:"link",relevance:0,begin:"\\]\\(",end:"\\)", +excludeBegin:!0,excludeEnd:!0},{className:"symbol",relevance:0,begin:"\\]\\[", +end:"\\]",excludeBegin:!0,excludeEnd:!0}]},s={className:"strong",contains:[], +variants:[{begin:/_{2}/,end:/_{2}/},{begin:/\*{2}/,end:/\*{2}/}]},c={ +className:"emphasis",contains:[],variants:[{begin:/\*(?!\*)/,end:/\*/},{ +begin:/_(?!_)/,end:/_/,relevance:0}]};s.contains.push(c),c.contains.push(s) +;let t=[a,i] +;return s.contains=s.contains.concat(t),c.contains=c.contains.concat(t), +t=t.concat(s,c),{name:"Markdown",aliases:["md","mkdown","mkd"],contains:[{ +className:"section",variants:[{begin:"^#{1,6}",end:"$",contains:t},{ +begin:"(?=^.+?\\n[=-]{2,}$)",contains:[{begin:"^[=-]*$"},{begin:"^",end:"\\n", +contains:t}]}]},a,{className:"bullet",begin:"^[ \t]*([*+-]|(\\d+\\.))(?=\\s+)", +end:"\\s+",excludeEnd:!0},s,c,{className:"quote",begin:"^>\\s+",contains:t, +end:"$"},{className:"code",variants:[{begin:"(`{3,})[^`](.|\\n)*?\\1`*[ ]*"},{ +begin:"(~{3,})[^~](.|\\n)*?\\1~*[ ]*"},{begin:"```",end:"```+[ ]*$"},{ +begin:"~~~",end:"~~~+[ ]*$"},{begin:"`.+?`"},{begin:"(?=^( {4}|\\t))", +contains:[{begin:"^( {4}|\\t)",end:"(\\n)$"}],relevance:0}]},{ +begin:"^[-\\*]{3,}",end:"$"},i,{begin:/^\[[^\n]+\]:/,returnBegin:!0,contains:[{ +className:"symbol",begin:/\[/,end:/\]/,excludeBegin:!0,excludeEnd:!0},{ +className:"link",begin:/:\s*/,end:/$/,excludeBegin:!0}]}]}}})());hljs.registerLanguage("bash",(()=>{"use strict";function e(...e){ +return e.map((e=>{return(s=e)?"string"==typeof s?s:s.source:null;var s +})).join("")}return s=>{const n={},t={begin:/\$\{/,end:/\}/,contains:["self",{ +begin:/:-/,contains:[n]}]};Object.assign(n,{className:"variable",variants:[{ +begin:e(/\$[\w\d#@][\w\d_]*/,"(?![\\w\\d])(?![$])")},t]});const a={ +className:"subst",begin:/\$\(/,end:/\)/,contains:[s.BACKSLASH_ESCAPE]},i={ +begin:/<<-?\s*(?=\w+)/,starts:{contains:[s.END_SAME_AS_BEGIN({begin:/(\w+)/, +end:/(\w+)/,className:"string"})]}},c={className:"string",begin:/"/,end:/"/, +contains:[s.BACKSLASH_ESCAPE,n,a]};a.contains.push(c);const o={begin:/\$\(\(/, +end:/\)\)/,contains:[{begin:/\d+#[0-9a-f]+/,className:"number"},s.NUMBER_MODE,n] +},r=s.SHEBANG({binary:"(fish|bash|zsh|sh|csh|ksh|tcsh|dash|scsh)",relevance:10 +}),l={className:"function",begin:/\w[\w\d_]*\s*\(\s*\)\s*\{/,returnBegin:!0, +contains:[s.inherit(s.TITLE_MODE,{begin:/\w[\w\d_]*/})],relevance:0};return{ +name:"Bash",aliases:["sh","zsh"],keywords:{$pattern:/\b[a-z._-]+\b/, +keyword:"if then else elif fi for while in do done case esac function", +literal:"true false", +built_in:"break cd continue eval exec exit export getopts hash pwd readonly return shift test times trap umask unset alias bind builtin caller command declare echo enable help let local logout mapfile printf read readarray source type typeset ulimit unalias set shopt autoload bg bindkey bye cap chdir clone comparguments compcall compctl compdescribe compfiles compgroups compquote comptags comptry compvalues dirs disable disown echotc echoti emulate fc fg float functions getcap getln history integer jobs kill limit log noglob popd print pushd pushln rehash sched setcap setopt stat suspend ttyctl unfunction unhash unlimit unsetopt vared wait whence where which zcompile zformat zftp zle zmodload zparseopts zprof zpty zregexparse zsocket zstyle ztcp" +},contains:[r,s.SHEBANG(),l,o,s.HASH_COMMENT_MODE,i,c,{className:"",begin:/\\"/ +},{className:"string",begin:/'/,end:/'/},n]}}})());hljs.registerLanguage("shell",(()=>{"use strict";return s=>({ +name:"Shell Session",aliases:["console"],contains:[{className:"meta", +begin:/^\s{0,3}[/~\w\d[\]()@-]*[>%$#]/,starts:{end:/[^\\](?=\s*$)/, +subLanguage:"bash"}}]})})());hljs.registerLanguage("json",(()=>{"use strict";return n=>{const e={ +literal:"true false null" +},i=[n.C_LINE_COMMENT_MODE,n.C_BLOCK_COMMENT_MODE],a=[n.QUOTE_STRING_MODE,n.C_NUMBER_MODE],l={ +end:",",endsWithParent:!0,excludeEnd:!0,contains:a,keywords:e},t={begin:/\{/, +end:/\}/,contains:[{className:"attr",begin:/"/,end:/"/, +contains:[n.BACKSLASH_ESCAPE],illegal:"\\n"},n.inherit(l,{begin:/:/ +})].concat(i),illegal:"\\S"},s={begin:"\\[",end:"\\]",contains:[n.inherit(l)], +illegal:"\\S"};return a.push(t,s),i.forEach((n=>{a.push(n)})),{name:"JSON", +contains:a,keywords:e,illegal:"\\S"}}})());hljs.registerLanguage("less",(()=>{"use strict" +;const e=["a","abbr","address","article","aside","audio","b","blockquote","body","button","canvas","caption","cite","code","dd","del","details","dfn","div","dl","dt","em","fieldset","figcaption","figure","footer","form","h1","h2","h3","h4","h5","h6","header","hgroup","html","i","iframe","img","input","ins","kbd","label","legend","li","main","mark","menu","nav","object","ol","p","q","quote","samp","section","span","strong","summary","sup","table","tbody","td","textarea","tfoot","th","thead","time","tr","ul","var","video"],t=["any-hover","any-pointer","aspect-ratio","color","color-gamut","color-index","device-aspect-ratio","device-height","device-width","display-mode","forced-colors","grid","height","hover","inverted-colors","monochrome","orientation","overflow-block","overflow-inline","pointer","prefers-color-scheme","prefers-contrast","prefers-reduced-motion","prefers-reduced-transparency","resolution","scan","scripting","update","width","min-width","max-width","min-height","max-height"],i=["active","any-link","blank","checked","current","default","defined","dir","disabled","drop","empty","enabled","first","first-child","first-of-type","fullscreen","future","focus","focus-visible","focus-within","has","host","host-context","hover","indeterminate","in-range","invalid","is","lang","last-child","last-of-type","left","link","local-link","not","nth-child","nth-col","nth-last-child","nth-last-col","nth-last-of-type","nth-of-type","only-child","only-of-type","optional","out-of-range","past","placeholder-shown","read-only","read-write","required","right","root","scope","target","target-within","user-invalid","valid","visited","where"],o=["after","backdrop","before","cue","cue-region","first-letter","first-line","grammar-error","marker","part","placeholder","selection","slotted","spelling-error"],n=["align-content","align-items","align-self","animation","animation-delay","animation-direction","animation-duration","animation-fill-mode","animation-iteration-count","animation-name","animation-play-state","animation-timing-function","auto","backface-visibility","background","background-attachment","background-clip","background-color","background-image","background-origin","background-position","background-repeat","background-size","border","border-bottom","border-bottom-color","border-bottom-left-radius","border-bottom-right-radius","border-bottom-style","border-bottom-width","border-collapse","border-color","border-image","border-image-outset","border-image-repeat","border-image-slice","border-image-source","border-image-width","border-left","border-left-color","border-left-style","border-left-width","border-radius","border-right","border-right-color","border-right-style","border-right-width","border-spacing","border-style","border-top","border-top-color","border-top-left-radius","border-top-right-radius","border-top-style","border-top-width","border-width","bottom","box-decoration-break","box-shadow","box-sizing","break-after","break-before","break-inside","caption-side","clear","clip","clip-path","color","column-count","column-fill","column-gap","column-rule","column-rule-color","column-rule-style","column-rule-width","column-span","column-width","columns","content","counter-increment","counter-reset","cursor","direction","display","empty-cells","filter","flex","flex-basis","flex-direction","flex-flow","flex-grow","flex-shrink","flex-wrap","float","font","font-display","font-family","font-feature-settings","font-kerning","font-language-override","font-size","font-size-adjust","font-smoothing","font-stretch","font-style","font-variant","font-variant-ligatures","font-variation-settings","font-weight","height","hyphens","icon","image-orientation","image-rendering","image-resolution","ime-mode","inherit","initial","justify-content","left","letter-spacing","line-height","list-style","list-style-image","list-style-position","list-style-type","margin","margin-bottom","margin-left","margin-right","margin-top","marks","mask","max-height","max-width","min-height","min-width","nav-down","nav-index","nav-left","nav-right","nav-up","none","normal","object-fit","object-position","opacity","order","orphans","outline","outline-color","outline-offset","outline-style","outline-width","overflow","overflow-wrap","overflow-x","overflow-y","padding","padding-bottom","padding-left","padding-right","padding-top","page-break-after","page-break-before","page-break-inside","perspective","perspective-origin","pointer-events","position","quotes","resize","right","src","tab-size","table-layout","text-align","text-align-last","text-decoration","text-decoration-color","text-decoration-line","text-decoration-style","text-indent","text-overflow","text-rendering","text-shadow","text-transform","text-underline-position","top","transform","transform-origin","transform-style","transition","transition-delay","transition-duration","transition-property","transition-timing-function","unicode-bidi","vertical-align","visibility","white-space","widows","width","word-break","word-spacing","word-wrap","z-index"].reverse(),r=i.concat(o) +;return a=>{const s=(e=>({IMPORTANT:{className:"meta",begin:"!important"}, +HEXCOLOR:{className:"number",begin:"#([a-fA-F0-9]{6}|[a-fA-F0-9]{3})"}, +ATTRIBUTE_SELECTOR_MODE:{className:"selector-attr",begin:/\[/,end:/\]/, +illegal:"$",contains:[e.APOS_STRING_MODE,e.QUOTE_STRING_MODE]} +}))(a),l=r,d="([\\w-]+|@\\{[\\w-]+\\})",c=[],g=[],b=e=>({className:"string", +begin:"~?"+e+".*?"+e}),m=(e,t,i)=>({className:e,begin:t,relevance:i}),u={ +$pattern:/[a-z-]+/,keyword:"and or not only",attribute:t.join(" ")},p={ +begin:"\\(",end:"\\)",contains:g,keywords:u,relevance:0} +;g.push(a.C_LINE_COMMENT_MODE,a.C_BLOCK_COMMENT_MODE,b("'"),b('"'),a.CSS_NUMBER_MODE,{ +begin:"(url|data-uri)\\(",starts:{className:"string",end:"[\\)\\n]", +excludeEnd:!0} +},s.HEXCOLOR,p,m("variable","@@?[\\w-]+",10),m("variable","@\\{[\\w-]+\\}"),m("built_in","~?`[^`]*?`"),{ +className:"attribute",begin:"[\\w-]+\\s*:",end:":",returnBegin:!0,excludeEnd:!0 +},s.IMPORTANT);const f=g.concat({begin:/\{/,end:/\}/,contains:c}),h={ +beginKeywords:"when",endsWithParent:!0,contains:[{beginKeywords:"and not" +}].concat(g)},w={begin:d+"\\s*:",returnBegin:!0,end:/[;}]/,relevance:0, +contains:[{begin:/-(webkit|moz|ms|o)-/},{className:"attribute", +begin:"\\b("+n.join("|")+")\\b",end:/(?=:)/,starts:{endsWithParent:!0, +illegal:"[<=$]",relevance:0,contains:g}}]},v={className:"keyword", +begin:"@(import|media|charset|font-face|(-[a-z]+-)?keyframes|supports|document|namespace|page|viewport|host)\\b", +starts:{end:"[;{}]",keywords:u,returnEnd:!0,contains:g,relevance:0}},y={ +className:"variable",variants:[{begin:"@[\\w-]+\\s*:",relevance:15},{ +begin:"@[\\w-]+"}],starts:{end:"[;}]",returnEnd:!0,contains:f}},k={variants:[{ +begin:"[\\.#:&\\[>]",end:"[;{}]"},{begin:d,end:/\{/}],returnBegin:!0, +returnEnd:!0,illegal:"[<='$\"]",relevance:0, +contains:[a.C_LINE_COMMENT_MODE,a.C_BLOCK_COMMENT_MODE,h,m("keyword","all\\b"),m("variable","@\\{[\\w-]+\\}"),{ +begin:"\\b("+e.join("|")+")\\b",className:"selector-tag" +},m("selector-tag",d+"%?",0),m("selector-id","#"+d),m("selector-class","\\."+d,0),m("selector-tag","&",0),s.ATTRIBUTE_SELECTOR_MODE,{ +className:"selector-pseudo",begin:":("+i.join("|")+")"},{ +className:"selector-pseudo",begin:"::("+o.join("|")+")"},{begin:"\\(",end:"\\)", +contains:f},{begin:"!important"}]},E={begin:`[\\w-]+:(:)?(${l.join("|")})`, +returnBegin:!0,contains:[k]} +;return c.push(a.C_LINE_COMMENT_MODE,a.C_BLOCK_COMMENT_MODE,v,y,E,w,k),{ +name:"Less",case_insensitive:!0,illegal:"[=>'/<($\"]",contains:c}}})());hljs.registerLanguage("nginx",(()=>{"use strict";return e=>{const n={ +className:"variable",variants:[{begin:/\$\d+/},{begin:/\$\{/,end:/\}/},{ +begin:/[$@]/+e.UNDERSCORE_IDENT_RE}]},a={endsWithParent:!0,keywords:{ +$pattern:"[a-z/_]+", +literal:"on off yes no true false none blocked debug info notice warn error crit select break last permanent redirect kqueue rtsig epoll poll /dev/poll" +},relevance:0,illegal:"=>",contains:[e.HASH_COMMENT_MODE,{className:"string", +contains:[e.BACKSLASH_ESCAPE,n],variants:[{begin:/"/,end:/"/},{begin:/'/,end:/'/ +}]},{begin:"([a-z]+):/",end:"\\s",endsWithParent:!0,excludeEnd:!0,contains:[n] +},{className:"regexp",contains:[e.BACKSLASH_ESCAPE,n],variants:[{begin:"\\s\\^", +end:"\\s|\\{|;",returnEnd:!0},{begin:"~\\*?\\s+",end:"\\s|\\{|;",returnEnd:!0},{ +begin:"\\*(\\.[a-z\\-]+)+"},{begin:"([a-z\\-]+\\.)+\\*"}]},{className:"number", +begin:"\\b\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}(:\\d{1,5})?\\b"},{ +className:"number",begin:"\\b\\d+[kKmMgGdshdwy]*\\b",relevance:0},n]};return{ +name:"Nginx config",aliases:["nginxconf"],contains:[e.HASH_COMMENT_MODE,{ +begin:e.UNDERSCORE_IDENT_RE+"\\s+\\{",returnBegin:!0,end:/\{/,contains:[{ +className:"section",begin:e.UNDERSCORE_IDENT_RE}],relevance:0},{ +begin:e.UNDERSCORE_IDENT_RE+"\\s",end:";|\\{",returnBegin:!0,contains:[{ +className:"attribute",begin:e.UNDERSCORE_IDENT_RE,starts:a}],relevance:0}], +illegal:"[^\\s\\}]"}}})());hljs.registerLanguage("diff",(()=>{"use strict";return e=>({name:"Diff", +aliases:["patch"],contains:[{className:"meta",relevance:10,variants:[{ +begin:/^@@ +-\d+,\d+ +\+\d+,\d+ +@@/},{begin:/^\*\*\* +\d+,\d+ +\*\*\*\*$/},{ +begin:/^--- +\d+,\d+ +----$/}]},{className:"comment",variants:[{begin:/Index: /, +end:/$/},{begin:/^index/,end:/$/},{begin:/={3,}/,end:/$/},{begin:/^-{3}/,end:/$/ +},{begin:/^\*{3} /,end:/$/},{begin:/^\+{3}/,end:/$/},{begin:/^\*{15}$/},{ +begin:/^diff --git/,end:/$/}]},{className:"addition",begin:/^\+/,end:/$/},{ +className:"deletion",begin:/^-/,end:/$/},{className:"addition",begin:/^!/, +end:/$/}]})})());hljs.registerLanguage("swift",(()=>{"use strict";function e(e){ +return e?"string"==typeof e?e:e.source:null}function n(e){return a("(?=",e,")")} +function a(...n){return n.map((n=>e(n))).join("")}function t(...n){ +return"("+n.map((n=>e(n))).join("|")+")"} +const i=e=>a(/\b/,e,/\w$/.test(e)?/\b/:/\B/),s=["Protocol","Type"].map(i),u=["init","self"].map(i),c=["Any","Self"],r=["associatedtype","async","await",/as\?/,/as!/,"as","break","case","catch","class","continue","convenience","default","defer","deinit","didSet","do","dynamic","else","enum","extension","fallthrough",/fileprivate\(set\)/,"fileprivate","final","for","func","get","guard","if","import","indirect","infix",/init\?/,/init!/,"inout",/internal\(set\)/,"internal","in","is","lazy","let","mutating","nonmutating",/open\(set\)/,"open","operator","optional","override","postfix","precedencegroup","prefix",/private\(set\)/,"private","protocol",/public\(set\)/,"public","repeat","required","rethrows","return","set","some","static","struct","subscript","super","switch","throws","throw",/try\?/,/try!/,"try","typealias",/unowned\(safe\)/,/unowned\(unsafe\)/,"unowned","var","weak","where","while","willSet"],o=["false","nil","true"],l=["assignment","associativity","higherThan","left","lowerThan","none","right"],m=["#colorLiteral","#column","#dsohandle","#else","#elseif","#endif","#error","#file","#fileID","#fileLiteral","#filePath","#function","#if","#imageLiteral","#keyPath","#line","#selector","#sourceLocation","#warn_unqualified_access","#warning"],d=["abs","all","any","assert","assertionFailure","debugPrint","dump","fatalError","getVaList","isKnownUniquelyReferenced","max","min","numericCast","pointwiseMax","pointwiseMin","precondition","preconditionFailure","print","readLine","repeatElement","sequence","stride","swap","swift_unboxFromSwiftValueWithType","transcode","type","unsafeBitCast","unsafeDowncast","withExtendedLifetime","withUnsafeMutablePointer","withUnsafePointer","withVaList","withoutActuallyEscaping","zip"],p=t(/[/=\-+!*%<>&|^~?]/,/[\u00A1-\u00A7]/,/[\u00A9\u00AB]/,/[\u00AC\u00AE]/,/[\u00B0\u00B1]/,/[\u00B6\u00BB\u00BF\u00D7\u00F7]/,/[\u2016-\u2017]/,/[\u2020-\u2027]/,/[\u2030-\u203E]/,/[\u2041-\u2053]/,/[\u2055-\u205E]/,/[\u2190-\u23FF]/,/[\u2500-\u2775]/,/[\u2794-\u2BFF]/,/[\u2E00-\u2E7F]/,/[\u3001-\u3003]/,/[\u3008-\u3020]/,/[\u3030]/),F=t(p,/[\u0300-\u036F]/,/[\u1DC0-\u1DFF]/,/[\u20D0-\u20FF]/,/[\uFE00-\uFE0F]/,/[\uFE20-\uFE2F]/),b=a(p,F,"*"),h=t(/[a-zA-Z_]/,/[\u00A8\u00AA\u00AD\u00AF\u00B2-\u00B5\u00B7-\u00BA]/,/[\u00BC-\u00BE\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u00FF]/,/[\u0100-\u02FF\u0370-\u167F\u1681-\u180D\u180F-\u1DBF]/,/[\u1E00-\u1FFF]/,/[\u200B-\u200D\u202A-\u202E\u203F-\u2040\u2054\u2060-\u206F]/,/[\u2070-\u20CF\u2100-\u218F\u2460-\u24FF\u2776-\u2793]/,/[\u2C00-\u2DFF\u2E80-\u2FFF]/,/[\u3004-\u3007\u3021-\u302F\u3031-\u303F\u3040-\uD7FF]/,/[\uF900-\uFD3D\uFD40-\uFDCF\uFDF0-\uFE1F\uFE30-\uFE44]/,/[\uFE47-\uFEFE\uFF00-\uFFFD]/),f=t(h,/\d/,/[\u0300-\u036F\u1DC0-\u1DFF\u20D0-\u20FF\uFE20-\uFE2F]/),w=a(h,f,"*"),y=a(/[A-Z]/,f,"*"),g=["autoclosure",a(/convention\(/,t("swift","block","c"),/\)/),"discardableResult","dynamicCallable","dynamicMemberLookup","escaping","frozen","GKInspectable","IBAction","IBDesignable","IBInspectable","IBOutlet","IBSegueAction","inlinable","main","nonobjc","NSApplicationMain","NSCopying","NSManaged",a(/objc\(/,w,/\)/),"objc","objcMembers","propertyWrapper","requires_stored_property_inits","testable","UIApplicationMain","unknown","usableFromInline"],E=["iOS","iOSApplicationExtension","macOS","macOSApplicationExtension","macCatalyst","macCatalystApplicationExtension","watchOS","watchOSApplicationExtension","tvOS","tvOSApplicationExtension","swift"] +;return e=>{const p={match:/\s+/,relevance:0},h=e.COMMENT("/\\*","\\*/",{ +contains:["self"]}),v=[e.C_LINE_COMMENT_MODE,h],N={className:"keyword", +begin:a(/\./,n(t(...s,...u))),end:t(...s,...u),excludeBegin:!0},A={ +match:a(/\./,t(...r)),relevance:0 +},C=r.filter((e=>"string"==typeof e)).concat(["_|0"]),_={variants:[{ +className:"keyword", +match:t(...r.filter((e=>"string"!=typeof e)).concat(c).map(i),...u)}]},D={ +$pattern:t(/\b\w+/,/#\w+/),keyword:C.concat(m),literal:o},B=[N,A,_],k=[{ +match:a(/\./,t(...d)),relevance:0},{className:"built_in", +match:a(/\b/,t(...d),/(?=\()/)}],M={match:/->/,relevance:0},S=[M,{ +className:"operator",relevance:0,variants:[{match:b},{match:`\\.(\\.|${F})+`}] +}],x="([0-9a-fA-F]_*)+",I={className:"number",relevance:0,variants:[{ +match:"\\b(([0-9]_*)+)(\\.(([0-9]_*)+))?([eE][+-]?(([0-9]_*)+))?\\b"},{ +match:`\\b0x(${x})(\\.(${x}))?([pP][+-]?(([0-9]_*)+))?\\b`},{ +match:/\b0o([0-7]_*)+\b/},{match:/\b0b([01]_*)+\b/}]},O=(e="")=>({ +className:"subst",variants:[{match:a(/\\/,e,/[0\\tnr"']/)},{ +match:a(/\\/,e,/u\{[0-9a-fA-F]{1,8}\}/)}]}),T=(e="")=>({className:"subst", +match:a(/\\/,e,/[\t ]*(?:[\r\n]|\r\n)/)}),L=(e="")=>({className:"subst", +label:"interpol",begin:a(/\\/,e,/\(/),end:/\)/}),P=(e="")=>({begin:a(e,/"""/), +end:a(/"""/,e),contains:[O(e),T(e),L(e)]}),$=(e="")=>({begin:a(e,/"/), +end:a(/"/,e),contains:[O(e),L(e)]}),K={className:"string", +variants:[P(),P("#"),P("##"),P("###"),$(),$("#"),$("##"),$("###")]},j={ +match:a(/`/,w,/`/)},z=[j,{className:"variable",match:/\$\d+/},{ +className:"variable",match:`\\$${f}+`}],q=[{match:/(@|#)available/, +className:"keyword",starts:{contains:[{begin:/\(/,end:/\)/,keywords:E, +contains:[...S,I,K]}]}},{className:"keyword",match:a(/@/,t(...g))},{ +className:"meta",match:a(/@/,w)}],U={match:n(/\b[A-Z]/),relevance:0,contains:[{ +className:"type", +match:a(/(AV|CA|CF|CG|CI|CL|CM|CN|CT|MK|MP|MTK|MTL|NS|SCN|SK|UI|WK|XC)/,f,"+") +},{className:"type",match:y,relevance:0},{match:/[?!]+/,relevance:0},{ +match:/\.\.\./,relevance:0},{match:a(/\s+&\s+/,n(y)),relevance:0}]},Z={ +begin://,keywords:D,contains:[...v,...B,...q,M,U]};U.contains.push(Z) +;const G={begin:/\(/,end:/\)/,relevance:0,keywords:D,contains:["self",{ +match:a(w,/\s*:/),keywords:"_|0",relevance:0 +},...v,...B,...k,...S,I,K,...z,...q,U]},H={beginKeywords:"func",contains:[{ +className:"title",match:t(j.match,w,b),endsParent:!0,relevance:0},p]},R={ +begin://,contains:[...v,U]},V={begin:/\(/,end:/\)/,keywords:D, +contains:[{begin:t(n(a(w,/\s*:/)),n(a(w,/\s+/,w,/\s*:/))),end:/:/,relevance:0, +contains:[{className:"keyword",match:/\b_\b/},{className:"params",match:w}] +},...v,...B,...S,I,K,...q,U,G],endsParent:!0,illegal:/["']/},W={ +className:"function",match:n(/\bfunc\b/),contains:[H,R,V,p],illegal:[/\[/,/%/] +},X={className:"function",match:/\b(subscript|init[?!]?)\s*(?=[<(])/,keywords:{ +keyword:"subscript init init? init!",$pattern:/\w+[?!]?/},contains:[R,V,p], +illegal:/\[|%/},J={beginKeywords:"operator",end:e.MATCH_NOTHING_RE,contains:[{ +className:"title",match:b,endsParent:!0,relevance:0}]},Q={ +beginKeywords:"precedencegroup",end:e.MATCH_NOTHING_RE,contains:[{ +className:"title",match:y,relevance:0},{begin:/{/,end:/}/,relevance:0, +endsParent:!0,keywords:[...l,...o],contains:[U]}]};for(const e of K.variants){ +const n=e.contains.find((e=>"interpol"===e.label));n.keywords=D +;const a=[...B,...k,...S,I,K,...z];n.contains=[...a,{begin:/\(/,end:/\)/, +contains:["self",...a]}]}return{name:"Swift",keywords:D,contains:[...v,W,X,{ +className:"class",beginKeywords:"struct protocol class extension enum", +end:"\\{",excludeEnd:!0,keywords:D,contains:[e.inherit(e.TITLE_MODE,{ +begin:/[A-Za-z$_][\u00C0-\u02B80-9A-Za-z$_]*/}),...B]},J,Q,{ +beginKeywords:"import",end:/$/,contains:[...v],relevance:0 +},...B,...k,...S,I,K,...z,...q,U,G]}}})());hljs.registerLanguage("php",(()=>{"use strict";return e=>{const r={ +className:"variable", +begin:"\\$+[a-zA-Z_\x7f-\xff][a-zA-Z0-9_\x7f-\xff]*(?![A-Za-z0-9])(?![$])"},t={ +className:"meta",variants:[{begin:/<\?php/,relevance:10},{begin:/<\?[=]?/},{ +begin:/\?>/}]},a={className:"subst",variants:[{begin:/\$\w+/},{begin:/\{\$/, +end:/\}/}]},n=e.inherit(e.APOS_STRING_MODE,{illegal:null +}),i=e.inherit(e.QUOTE_STRING_MODE,{illegal:null, +contains:e.QUOTE_STRING_MODE.contains.concat(a)}),o=e.END_SAME_AS_BEGIN({ +begin:/<<<[ \t]*(\w+)\n/,end:/[ \t]*(\w+)\b/, +contains:e.QUOTE_STRING_MODE.contains.concat(a)}),l={className:"string", +contains:[e.BACKSLASH_ESCAPE,t],variants:[e.inherit(n,{begin:"b'",end:"'" +}),e.inherit(i,{begin:'b"',end:'"'}),i,n,o]},s={className:"number",variants:[{ +begin:"\\b0b[01]+(?:_[01]+)*\\b"},{begin:"\\b0o[0-7]+(?:_[0-7]+)*\\b"},{ +begin:"\\b0x[\\da-f]+(?:_[\\da-f]+)*\\b"},{ +begin:"(?:\\b\\d+(?:_\\d+)*(\\.(?:\\d+(?:_\\d+)*))?|\\B\\.\\d+)(?:e[+-]?\\d+)?" +}],relevance:0},c={ +keyword:"__CLASS__ __DIR__ __FILE__ __FUNCTION__ __LINE__ __METHOD__ __NAMESPACE__ __TRAIT__ die echo exit include include_once print require require_once array abstract and as binary bool boolean break callable case catch class clone const continue declare default do double else elseif empty enddeclare endfor endforeach endif endswitch endwhile enum eval extends final finally float for foreach from global goto if implements instanceof insteadof int integer interface isset iterable list match|0 mixed new object or private protected public real return string switch throw trait try unset use var void while xor yield", +literal:"false null true", +built_in:"Error|0 AppendIterator ArgumentCountError ArithmeticError ArrayIterator ArrayObject AssertionError BadFunctionCallException BadMethodCallException CachingIterator CallbackFilterIterator CompileError Countable DirectoryIterator DivisionByZeroError DomainException EmptyIterator ErrorException Exception FilesystemIterator FilterIterator GlobIterator InfiniteIterator InvalidArgumentException IteratorIterator LengthException LimitIterator LogicException MultipleIterator NoRewindIterator OutOfBoundsException OutOfRangeException OuterIterator OverflowException ParentIterator ParseError RangeException RecursiveArrayIterator RecursiveCachingIterator RecursiveCallbackFilterIterator RecursiveDirectoryIterator RecursiveFilterIterator RecursiveIterator RecursiveIteratorIterator RecursiveRegexIterator RecursiveTreeIterator RegexIterator RuntimeException SeekableIterator SplDoublyLinkedList SplFileInfo SplFileObject SplFixedArray SplHeap SplMaxHeap SplMinHeap SplObjectStorage SplObserver SplObserver SplPriorityQueue SplQueue SplStack SplSubject SplSubject SplTempFileObject TypeError UnderflowException UnexpectedValueException UnhandledMatchError ArrayAccess Closure Generator Iterator IteratorAggregate Serializable Stringable Throwable Traversable WeakReference WeakMap Directory __PHP_Incomplete_Class parent php_user_filter self static stdClass" +};return{aliases:["php3","php4","php5","php6","php7","php8"], +case_insensitive:!0,keywords:c, +contains:[e.HASH_COMMENT_MODE,e.COMMENT("//","$",{contains:[t] +}),e.COMMENT("/\\*","\\*/",{contains:[{className:"doctag",begin:"@[A-Za-z]+"}] +}),e.COMMENT("__halt_compiler.+?;",!1,{endsWithParent:!0, +keywords:"__halt_compiler"}),t,{className:"keyword",begin:/\$this\b/},r,{ +begin:/(::|->)+[a-zA-Z_\x7f-\xff][a-zA-Z0-9_\x7f-\xff]*/},{className:"function", +relevance:0,beginKeywords:"fn function",end:/[;{]/,excludeEnd:!0, +illegal:"[$%\\[]",contains:[{beginKeywords:"use"},e.UNDERSCORE_TITLE_MODE,{ +begin:"=>",endsParent:!0},{className:"params",begin:"\\(",end:"\\)", +excludeBegin:!0,excludeEnd:!0,keywords:c, +contains:["self",r,e.C_BLOCK_COMMENT_MODE,l,s]}]},{className:"class",variants:[{ +beginKeywords:"enum",illegal:/[($"]/},{beginKeywords:"class interface trait", +illegal:/[:($"]/}],relevance:0,end:/\{/,excludeEnd:!0,contains:[{ +beginKeywords:"extends implements"},e.UNDERSCORE_TITLE_MODE]},{ +beginKeywords:"namespace",relevance:0,end:";",illegal:/[.']/, +contains:[e.UNDERSCORE_TITLE_MODE]},{beginKeywords:"use",relevance:0,end:";", +contains:[e.UNDERSCORE_TITLE_MODE]},l,s]}}})());hljs.registerLanguage("php-template",(()=>{"use strict";return n=>({ +name:"PHP template",subLanguage:"xml",contains:[{begin:/<\?(php|=)?/,end:/\?>/, +subLanguage:"php",contains:[{begin:"/\\*",end:"\\*/",skip:!0},{begin:'b"', +end:'"',skip:!0},{begin:"b'",end:"'",skip:!0},n.inherit(n.APOS_STRING_MODE,{ +illegal:null,className:null,contains:null,skip:!0 +}),n.inherit(n.QUOTE_STRING_MODE,{illegal:null,className:null,contains:null, +skip:!0})]}]})})());hljs.registerLanguage("ruby",(()=>{"use strict";function e(...e){ +return e.map((e=>{return(n=e)?"string"==typeof n?n:n.source:null;var n +})).join("")}return n=>{ +const a="([a-zA-Z_]\\w*[!?=]?|[-+~]@|<<|>>|=~|===?|<=>|[<>]=?|\\*\\*|[-/+%^&*~`|]|\\[\\]=?)",i={ +keyword:"and then defined module in return redo if BEGIN retry end for self when next until do begin unless END rescue else break undef not super class case require yield alias while ensure elsif or include attr_reader attr_writer attr_accessor __FILE__", +built_in:"proc lambda",literal:"true false nil"},s={className:"doctag", +begin:"@[A-Za-z]+"},r={begin:"#<",end:">"},b=[n.COMMENT("#","$",{contains:[s] +}),n.COMMENT("^=begin","^=end",{contains:[s],relevance:10 +}),n.COMMENT("^__END__","\\n$")],c={className:"subst",begin:/#\{/,end:/\}/, +keywords:i},t={className:"string",contains:[n.BACKSLASH_ESCAPE,c],variants:[{ +begin:/'/,end:/'/},{begin:/"/,end:/"/},{begin:/`/,end:/`/},{begin:/%[qQwWx]?\(/, +end:/\)/},{begin:/%[qQwWx]?\[/,end:/\]/},{begin:/%[qQwWx]?\{/,end:/\}/},{ +begin:/%[qQwWx]?/},{begin:/%[qQwWx]?\//,end:/\//},{begin:/%[qQwWx]?%/, +end:/%/},{begin:/%[qQwWx]?-/,end:/-/},{begin:/%[qQwWx]?\|/,end:/\|/},{ +begin:/\B\?(\\\d{1,3})/},{begin:/\B\?(\\x[A-Fa-f0-9]{1,2})/},{ +begin:/\B\?(\\u\{?[A-Fa-f0-9]{1,6}\}?)/},{ +begin:/\B\?(\\M-\\C-|\\M-\\c|\\c\\M-|\\M-|\\C-\\M-)[\x20-\x7e]/},{ +begin:/\B\?\\(c|C-)[\x20-\x7e]/},{begin:/\B\?\\?\S/},{ +begin:/<<[-~]?'?(\w+)\n(?:[^\n]*\n)*?\s*\1\b/,returnBegin:!0,contains:[{ +begin:/<<[-~]?'?/},n.END_SAME_AS_BEGIN({begin:/(\w+)/,end:/(\w+)/, +contains:[n.BACKSLASH_ESCAPE,c]})]}]},g="[0-9](_?[0-9])*",d={className:"number", +relevance:0,variants:[{ +begin:`\\b([1-9](_?[0-9])*|0)(\\.(${g}))?([eE][+-]?(${g})|r)?i?\\b`},{ +begin:"\\b0[dD][0-9](_?[0-9])*r?i?\\b"},{begin:"\\b0[bB][0-1](_?[0-1])*r?i?\\b" +},{begin:"\\b0[oO][0-7](_?[0-7])*r?i?\\b"},{ +begin:"\\b0[xX][0-9a-fA-F](_?[0-9a-fA-F])*r?i?\\b"},{ +begin:"\\b0(_?[0-7])+r?i?\\b"}]},l={className:"params",begin:"\\(",end:"\\)", +endsParent:!0,keywords:i},o=[t,{className:"class",beginKeywords:"class module", +end:"$|;",illegal:/=/,contains:[n.inherit(n.TITLE_MODE,{ +begin:"[A-Za-z_]\\w*(::\\w+)*(\\?|!)?"}),{begin:"<\\s*",contains:[{ +begin:"("+n.IDENT_RE+"::)?"+n.IDENT_RE,relevance:0}]}].concat(b)},{ +className:"function",begin:e(/def\s+/,(_=a+"\\s*(\\(|;|$)",e("(?=",_,")"))), +relevance:0,keywords:"def",end:"$|;",contains:[n.inherit(n.TITLE_MODE,{begin:a +}),l].concat(b)},{begin:n.IDENT_RE+"::"},{className:"symbol", +begin:n.UNDERSCORE_IDENT_RE+"(!|\\?)?:",relevance:0},{className:"symbol", +begin:":(?!\\s)",contains:[t,{begin:a}],relevance:0},d,{className:"variable", +begin:"(\\$\\W)|((\\$|@@?)(\\w+))(?=[^@$?])(?![A-Za-z])(?![@$?'])"},{ +className:"params",begin:/\|/,end:/\|/,relevance:0,keywords:i},{ +begin:"("+n.RE_STARTERS_RE+"|unless)\\s*",keywords:"unless",contains:[{ +className:"regexp",contains:[n.BACKSLASH_ESCAPE,c],illegal:/\n/,variants:[{ +begin:"/",end:"/[a-z]*"},{begin:/%r\{/,end:/\}[a-z]*/},{begin:"%r\\(", +end:"\\)[a-z]*"},{begin:"%r!",end:"![a-z]*"},{begin:"%r\\[",end:"\\][a-z]*"}] +}].concat(r,b),relevance:0}].concat(r,b);var _;c.contains=o,l.contains=o +;const E=[{begin:/^\s*=>/,starts:{end:"$",contains:o}},{className:"meta", +begin:"^([>?]>|[\\w#]+\\(\\w+\\):\\d+:\\d+>|(\\w+-)?\\d+\\.\\d+\\.\\d+(p\\d+)?[^\\d][^>]+>)(?=[ ])", +starts:{end:"$",contains:o}}];return b.unshift(r),{name:"Ruby", +aliases:["rb","gemspec","podspec","thor","irb"],keywords:i,illegal:/\/\*/, +contains:[n.SHEBANG({binary:"ruby"})].concat(E).concat(b).concat(o)}}})());hljs.registerLanguage("yaml",(()=>{"use strict";return e=>{ +var n="true false yes no null",a="[\\w#;/?:@&=+$,.~*'()[\\]]+",s={ +className:"string",relevance:0,variants:[{begin:/'/,end:/'/},{begin:/"/,end:/"/ +},{begin:/\S+/}],contains:[e.BACKSLASH_ESCAPE,{className:"template-variable", +variants:[{begin:/\{\{/,end:/\}\}/},{begin:/%\{/,end:/\}/}]}]},i=e.inherit(s,{ +variants:[{begin:/'/,end:/'/},{begin:/"/,end:/"/},{begin:/[^\s,{}[\]]+/}]}),l={ +end:",",endsWithParent:!0,excludeEnd:!0,keywords:n,relevance:0},t={begin:/\{/, +end:/\}/,contains:[l],illegal:"\\n",relevance:0},g={begin:"\\[",end:"\\]", +contains:[l],illegal:"\\n",relevance:0},b=[{className:"attr",variants:[{ +begin:"\\w[\\w :\\/.-]*:(?=[ \t]|$)"},{begin:'"\\w[\\w :\\/.-]*":(?=[ \t]|$)'},{ +begin:"'\\w[\\w :\\/.-]*':(?=[ \t]|$)"}]},{className:"meta",begin:"^---\\s*$", +relevance:10},{className:"string", +begin:"[\\|>]([1-9]?[+-])?[ ]*\\n( +)[^ ][^\\n]*\\n(\\2[^\\n]+\\n?)*"},{ +begin:"<%[%=-]?",end:"[%-]?%>",subLanguage:"ruby",excludeBegin:!0,excludeEnd:!0, +relevance:0},{className:"type",begin:"!\\w+!"+a},{className:"type", +begin:"!<"+a+">"},{className:"type",begin:"!"+a},{className:"type",begin:"!!"+a +},{className:"meta",begin:"&"+e.UNDERSCORE_IDENT_RE+"$"},{className:"meta", +begin:"\\*"+e.UNDERSCORE_IDENT_RE+"$"},{className:"bullet",begin:"-(?=[ ]|$)", +relevance:0},e.HASH_COMMENT_MODE,{beginKeywords:n,keywords:{literal:n}},{ +className:"number", +begin:"\\b[0-9]{4}(-[0-9][0-9]){0,2}([Tt \\t][0-9][0-9]?(:[0-9][0-9]){2})?(\\.[0-9]*)?([ \\t])*(Z|[-+][0-9][0-9]?(:[0-9][0-9])?)?\\b" +},{className:"number",begin:e.C_NUMBER_RE+"\\b",relevance:0},t,g,s],r=[...b] +;return r.pop(),r.push(i),l.contains=r,{name:"YAML",case_insensitive:!0, +aliases:["yml"],contains:b}}})());hljs.registerLanguage("r",(()=>{"use strict";function e(...e){return e.map((e=>{ +return(a=e)?"string"==typeof a?a:a.source:null;var a})).join("")}return a=>{ +const n=/(?:(?:[a-zA-Z]|\.[._a-zA-Z])[._a-zA-Z0-9]*)|\.(?!\d)/;return{name:"R", +illegal:/->/,keywords:{$pattern:n, +keyword:"function if in break next repeat else for while", +literal:"NULL NA TRUE FALSE Inf NaN NA_integer_|10 NA_real_|10 NA_character_|10 NA_complex_|10", +built_in:"LETTERS letters month.abb month.name pi T F abs acos acosh all any anyNA Arg as.call as.character as.complex as.double as.environment as.integer as.logical as.null.default as.numeric as.raw asin asinh atan atanh attr attributes baseenv browser c call ceiling class Conj cos cosh cospi cummax cummin cumprod cumsum digamma dim dimnames emptyenv exp expression floor forceAndCall gamma gc.time globalenv Im interactive invisible is.array is.atomic is.call is.character is.complex is.double is.environment is.expression is.finite is.function is.infinite is.integer is.language is.list is.logical is.matrix is.na is.name is.nan is.null is.numeric is.object is.pairlist is.raw is.recursive is.single is.symbol lazyLoadDBfetch length lgamma list log max min missing Mod names nargs nzchar oldClass on.exit pos.to.env proc.time prod quote range Re rep retracemem return round seq_along seq_len seq.int sign signif sin sinh sinpi sqrt standardGeneric substitute sum switch tan tanh tanpi tracemem trigamma trunc unclass untracemem UseMethod xtfrm" +},compilerExtensions:[(a,n)=>{if(!a.beforeMatch)return +;if(a.starts)throw Error("beforeMatch cannot be used with starts") +;const i=Object.assign({},a);Object.keys(a).forEach((e=>{delete a[e] +})),a.begin=e(i.beforeMatch,e("(?=",i.begin,")")),a.starts={relevance:0, +contains:[Object.assign(i,{endsParent:!0})]},a.relevance=0,delete i.beforeMatch +}],contains:[a.COMMENT(/#'/,/$/,{contains:[{className:"doctag", +begin:"@examples",starts:{contains:[{begin:/\n/},{begin:/#'\s*(?=@[a-zA-Z]+)/, +endsParent:!0},{begin:/#'/,end:/$/,excludeBegin:!0}]}},{className:"doctag", +begin:"@param",end:/$/,contains:[{className:"variable",variants:[{begin:n},{ +begin:/`(?:\\.|[^`\\])+`/}],endsParent:!0}]},{className:"doctag", +begin:/@[a-zA-Z]+/},{className:"meta-keyword",begin:/\\[a-zA-Z]+/}] +}),a.HASH_COMMENT_MODE,{className:"string",contains:[a.BACKSLASH_ESCAPE], +variants:[a.END_SAME_AS_BEGIN({begin:/[rR]"(-*)\(/,end:/\)(-*)"/ +}),a.END_SAME_AS_BEGIN({begin:/[rR]"(-*)\{/,end:/\}(-*)"/ +}),a.END_SAME_AS_BEGIN({begin:/[rR]"(-*)\[/,end:/\](-*)"/ +}),a.END_SAME_AS_BEGIN({begin:/[rR]'(-*)\(/,end:/\)(-*)'/ +}),a.END_SAME_AS_BEGIN({begin:/[rR]'(-*)\{/,end:/\}(-*)'/ +}),a.END_SAME_AS_BEGIN({begin:/[rR]'(-*)\[/,end:/\](-*)'/}),{begin:'"',end:'"', +relevance:0},{begin:"'",end:"'",relevance:0}]},{className:"number",relevance:0, +beforeMatch:/([^a-zA-Z0-9._])/,variants:[{ +match:/0[xX][0-9a-fA-F]+\.[0-9a-fA-F]*[pP][+-]?\d+i?/},{ +match:/0[xX][0-9a-fA-F]+([pP][+-]?\d+)?[Li]?/},{ +match:/(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?[Li]?/}]},{begin:"%",end:"%"},{ +begin:e(/[a-zA-Z][a-zA-Z_0-9]*/,"\\s+<-\\s+")},{begin:"`",end:"`",contains:[{ +begin:/\\./}]}]}}})());hljs.registerLanguage("scss",(()=>{"use strict" +;const e=["a","abbr","address","article","aside","audio","b","blockquote","body","button","canvas","caption","cite","code","dd","del","details","dfn","div","dl","dt","em","fieldset","figcaption","figure","footer","form","h1","h2","h3","h4","h5","h6","header","hgroup","html","i","iframe","img","input","ins","kbd","label","legend","li","main","mark","menu","nav","object","ol","p","q","quote","samp","section","span","strong","summary","sup","table","tbody","td","textarea","tfoot","th","thead","time","tr","ul","var","video"],t=["any-hover","any-pointer","aspect-ratio","color","color-gamut","color-index","device-aspect-ratio","device-height","device-width","display-mode","forced-colors","grid","height","hover","inverted-colors","monochrome","orientation","overflow-block","overflow-inline","pointer","prefers-color-scheme","prefers-contrast","prefers-reduced-motion","prefers-reduced-transparency","resolution","scan","scripting","update","width","min-width","max-width","min-height","max-height"],i=["active","any-link","blank","checked","current","default","defined","dir","disabled","drop","empty","enabled","first","first-child","first-of-type","fullscreen","future","focus","focus-visible","focus-within","has","host","host-context","hover","indeterminate","in-range","invalid","is","lang","last-child","last-of-type","left","link","local-link","not","nth-child","nth-col","nth-last-child","nth-last-col","nth-last-of-type","nth-of-type","only-child","only-of-type","optional","out-of-range","past","placeholder-shown","read-only","read-write","required","right","root","scope","target","target-within","user-invalid","valid","visited","where"],o=["after","backdrop","before","cue","cue-region","first-letter","first-line","grammar-error","marker","part","placeholder","selection","slotted","spelling-error"],r=["align-content","align-items","align-self","animation","animation-delay","animation-direction","animation-duration","animation-fill-mode","animation-iteration-count","animation-name","animation-play-state","animation-timing-function","auto","backface-visibility","background","background-attachment","background-clip","background-color","background-image","background-origin","background-position","background-repeat","background-size","border","border-bottom","border-bottom-color","border-bottom-left-radius","border-bottom-right-radius","border-bottom-style","border-bottom-width","border-collapse","border-color","border-image","border-image-outset","border-image-repeat","border-image-slice","border-image-source","border-image-width","border-left","border-left-color","border-left-style","border-left-width","border-radius","border-right","border-right-color","border-right-style","border-right-width","border-spacing","border-style","border-top","border-top-color","border-top-left-radius","border-top-right-radius","border-top-style","border-top-width","border-width","bottom","box-decoration-break","box-shadow","box-sizing","break-after","break-before","break-inside","caption-side","clear","clip","clip-path","color","column-count","column-fill","column-gap","column-rule","column-rule-color","column-rule-style","column-rule-width","column-span","column-width","columns","content","counter-increment","counter-reset","cursor","direction","display","empty-cells","filter","flex","flex-basis","flex-direction","flex-flow","flex-grow","flex-shrink","flex-wrap","float","font","font-display","font-family","font-feature-settings","font-kerning","font-language-override","font-size","font-size-adjust","font-smoothing","font-stretch","font-style","font-variant","font-variant-ligatures","font-variation-settings","font-weight","height","hyphens","icon","image-orientation","image-rendering","image-resolution","ime-mode","inherit","initial","justify-content","left","letter-spacing","line-height","list-style","list-style-image","list-style-position","list-style-type","margin","margin-bottom","margin-left","margin-right","margin-top","marks","mask","max-height","max-width","min-height","min-width","nav-down","nav-index","nav-left","nav-right","nav-up","none","normal","object-fit","object-position","opacity","order","orphans","outline","outline-color","outline-offset","outline-style","outline-width","overflow","overflow-wrap","overflow-x","overflow-y","padding","padding-bottom","padding-left","padding-right","padding-top","page-break-after","page-break-before","page-break-inside","perspective","perspective-origin","pointer-events","position","quotes","resize","right","src","tab-size","table-layout","text-align","text-align-last","text-decoration","text-decoration-color","text-decoration-line","text-decoration-style","text-indent","text-overflow","text-rendering","text-shadow","text-transform","text-underline-position","top","transform","transform-origin","transform-style","transition","transition-delay","transition-duration","transition-property","transition-timing-function","unicode-bidi","vertical-align","visibility","white-space","widows","width","word-break","word-spacing","word-wrap","z-index"].reverse() +;return a=>{const n=(e=>({IMPORTANT:{className:"meta",begin:"!important"}, +HEXCOLOR:{className:"number",begin:"#([a-fA-F0-9]{6}|[a-fA-F0-9]{3})"}, +ATTRIBUTE_SELECTOR_MODE:{className:"selector-attr",begin:/\[/,end:/\]/, +illegal:"$",contains:[e.APOS_STRING_MODE,e.QUOTE_STRING_MODE]} +}))(a),l=o,s=i,d="@[a-z-]+",c={className:"variable", +begin:"(\\$[a-zA-Z-][a-zA-Z0-9_-]*)\\b"};return{name:"SCSS",case_insensitive:!0, +illegal:"[=/|']",contains:[a.C_LINE_COMMENT_MODE,a.C_BLOCK_COMMENT_MODE,{ +className:"selector-id",begin:"#[A-Za-z0-9_-]+",relevance:0},{ +className:"selector-class",begin:"\\.[A-Za-z0-9_-]+",relevance:0 +},n.ATTRIBUTE_SELECTOR_MODE,{className:"selector-tag", +begin:"\\b("+e.join("|")+")\\b",relevance:0},{className:"selector-pseudo", +begin:":("+s.join("|")+")"},{className:"selector-pseudo", +begin:"::("+l.join("|")+")"},c,{begin:/\(/,end:/\)/,contains:[a.CSS_NUMBER_MODE] +},{className:"attribute",begin:"\\b("+r.join("|")+")\\b"},{ +begin:"\\b(whitespace|wait|w-resize|visible|vertical-text|vertical-ideographic|uppercase|upper-roman|upper-alpha|underline|transparent|top|thin|thick|text|text-top|text-bottom|tb-rl|table-header-group|table-footer-group|sw-resize|super|strict|static|square|solid|small-caps|separate|se-resize|scroll|s-resize|rtl|row-resize|ridge|right|repeat|repeat-y|repeat-x|relative|progress|pointer|overline|outside|outset|oblique|nowrap|not-allowed|normal|none|nw-resize|no-repeat|no-drop|newspaper|ne-resize|n-resize|move|middle|medium|ltr|lr-tb|lowercase|lower-roman|lower-alpha|loose|list-item|line|line-through|line-edge|lighter|left|keep-all|justify|italic|inter-word|inter-ideograph|inside|inset|inline|inline-block|inherit|inactive|ideograph-space|ideograph-parenthesis|ideograph-numeric|ideograph-alpha|horizontal|hidden|help|hand|groove|fixed|ellipsis|e-resize|double|dotted|distribute|distribute-space|distribute-letter|distribute-all-lines|disc|disabled|default|decimal|dashed|crosshair|collapse|col-resize|circle|char|center|capitalize|break-word|break-all|bottom|both|bolder|bold|block|bidi-override|below|baseline|auto|always|all-scroll|absolute|table|table-cell)\\b" +},{begin:":",end:";", +contains:[c,n.HEXCOLOR,a.CSS_NUMBER_MODE,a.QUOTE_STRING_MODE,a.APOS_STRING_MODE,n.IMPORTANT] +},{begin:"@(page|font-face)",lexemes:d,keywords:"@page @font-face"},{begin:"@", +end:"[{;]",returnBegin:!0,keywords:{$pattern:/[a-z-]+/, +keyword:"and or not only",attribute:t.join(" ")},contains:[{begin:d, +className:"keyword"},{begin:/[a-z-]+(?=:)/,className:"attribute" +},c,a.QUOTE_STRING_MODE,a.APOS_STRING_MODE,n.HEXCOLOR,a.CSS_NUMBER_MODE]}]}} +})());hljs.registerLanguage("lua",(()=>{"use strict";return e=>{ +const t="\\[=*\\[",a="\\]=*\\]",n={begin:t,end:a,contains:["self"] +},o=[e.COMMENT("--(?!\\[=*\\[)","$"),e.COMMENT("--\\[=*\\[",a,{contains:[n], +relevance:10})];return{name:"Lua",keywords:{$pattern:e.UNDERSCORE_IDENT_RE, +literal:"true false nil", +keyword:"and break do else elseif end for goto if in local not or repeat return then until while", +built_in:"_G _ENV _VERSION __index __newindex __mode __call __metatable __tostring __len __gc __add __sub __mul __div __mod __pow __concat __unm __eq __lt __le assert collectgarbage dofile error getfenv getmetatable ipairs load loadfile loadstring module next pairs pcall print rawequal rawget rawset require select setfenv setmetatable tonumber tostring type unpack xpcall arg self coroutine resume yield status wrap create running debug getupvalue debug sethook getmetatable gethook setmetatable setlocal traceback setfenv getinfo setupvalue getlocal getregistry getfenv io lines write close flush open output type read stderr stdin input stdout popen tmpfile math log max acos huge ldexp pi cos tanh pow deg tan cosh sinh random randomseed frexp ceil floor rad abs sqrt modf asin min mod fmod log10 atan2 exp sin atan os exit setlocale date getenv difftime remove time clock tmpname rename execute package preload loadlib loaded loaders cpath config path seeall string sub upper len gfind rep find match char dump gmatch reverse byte format gsub lower table setn insert getn foreachi maxn foreach concat sort remove" +},contains:o.concat([{className:"function",beginKeywords:"function",end:"\\)", +contains:[e.inherit(e.TITLE_MODE,{ +begin:"([_a-zA-Z]\\w*\\.)*([_a-zA-Z]\\w*:)?[_a-zA-Z]\\w*"}),{className:"params", +begin:"\\(",endsWithParent:!0,contains:o}].concat(o) +},e.C_NUMBER_MODE,e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,{className:"string", +begin:t,end:a,contains:[n],relevance:5}])}}})());hljs.registerLanguage("vbnet",(()=>{"use strict";function e(e){ +return e?"string"==typeof e?e:e.source:null}function n(...n){ +return n.map((n=>e(n))).join("")}function t(...n){ +return"("+n.map((n=>e(n))).join("|")+")"}return e=>{ +const a=/\d{1,2}\/\d{1,2}\/\d{4}/,i=/\d{4}-\d{1,2}-\d{1,2}/,s=/(\d|1[012])(:\d+){0,2} *(AM|PM)/,r=/\d{1,2}(:\d{1,2}){1,2}/,o={ +className:"literal",variants:[{begin:n(/# */,t(i,a),/ *#/)},{ +begin:n(/# */,r,/ *#/)},{begin:n(/# */,s,/ *#/)},{ +begin:n(/# */,t(i,a),/ +/,t(s,r),/ *#/)}]},l=e.COMMENT(/'''/,/$/,{contains:[{ +className:"doctag",begin:/<\/?/,end:/>/}]}),c=e.COMMENT(null,/$/,{variants:[{ +begin:/'/},{begin:/([\t ]|^)REM(?=\s)/}]});return{name:"Visual Basic .NET", +aliases:["vb"],case_insensitive:!0,classNameAliases:{label:"symbol"},keywords:{ +keyword:"addhandler alias aggregate ansi as async assembly auto binary by byref byval call case catch class compare const continue custom declare default delegate dim distinct do each equals else elseif end enum erase error event exit explicit finally for friend from function get global goto group handles if implements imports in inherits interface into iterator join key let lib loop me mid module mustinherit mustoverride mybase myclass namespace narrowing new next notinheritable notoverridable of off on operator option optional order overloads overridable overrides paramarray partial preserve private property protected public raiseevent readonly redim removehandler resume return select set shadows shared skip static step stop structure strict sub synclock take text then throw to try unicode until using when where while widening with withevents writeonly yield", +built_in:"addressof and andalso await directcast gettype getxmlnamespace is isfalse isnot istrue like mod nameof new not or orelse trycast typeof xor cbool cbyte cchar cdate cdbl cdec cint clng cobj csbyte cshort csng cstr cuint culng cushort", +type:"boolean byte char date decimal double integer long object sbyte short single string uinteger ulong ushort", +literal:"true false nothing"}, +illegal:"//|\\{|\\}|endif|gosub|variant|wend|^\\$ ",contains:[{ +className:"string",begin:/"(""|[^/n])"C\b/},{className:"string",begin:/"/, +end:/"/,illegal:/\n/,contains:[{begin:/""/}]},o,{className:"number",relevance:0, +variants:[{begin:/\b\d[\d_]*((\.[\d_]+(E[+-]?[\d_]+)?)|(E[+-]?[\d_]+))[RFD@!#]?/ +},{begin:/\b\d[\d_]*((U?[SIL])|[%&])?/},{begin:/&H[\dA-F_]+((U?[SIL])|[%&])?/},{ +begin:/&O[0-7_]+((U?[SIL])|[%&])?/},{begin:/&B[01_]+((U?[SIL])|[%&])?/}]},{ +className:"label",begin:/^\w+:/},l,c,{className:"meta", +begin:/[\t ]*#(const|disable|else|elseif|enable|end|externalsource|if|region)\b/, +end:/$/,keywords:{ +"meta-keyword":"const disable else elseif enable end externalsource if region then" +},contains:[c]}]}}})());hljs.registerLanguage("rust",(()=>{"use strict";return e=>{ +const n="([ui](8|16|32|64|128|size)|f(32|64))?",t="drop i8 i16 i32 i64 i128 isize u8 u16 u32 u64 u128 usize f32 f64 str char bool Box Option Result String Vec Copy Send Sized Sync Drop Fn FnMut FnOnce ToOwned Clone Debug PartialEq PartialOrd Eq Ord AsRef AsMut Into From Default Iterator Extend IntoIterator DoubleEndedIterator ExactSizeIterator SliceConcatExt ToString assert! assert_eq! bitflags! bytes! cfg! col! concat! concat_idents! debug_assert! debug_assert_eq! env! panic! file! format! format_args! include_bin! include_str! line! local_data_key! module_path! option_env! print! println! select! stringify! try! unimplemented! unreachable! vec! write! writeln! macro_rules! assert_ne! debug_assert_ne!" +;return{name:"Rust",aliases:["rs"],keywords:{$pattern:e.IDENT_RE+"!?", +keyword:"abstract as async await become box break const continue crate do dyn else enum extern false final fn for if impl in let loop macro match mod move mut override priv pub ref return self Self static struct super trait true try type typeof unsafe unsized use virtual where while yield", +literal:"true false Some None Ok Err",built_in:t},illegal:""}]}}})());hljs.registerLanguage("css",(()=>{"use strict" +;const e=["a","abbr","address","article","aside","audio","b","blockquote","body","button","canvas","caption","cite","code","dd","del","details","dfn","div","dl","dt","em","fieldset","figcaption","figure","footer","form","h1","h2","h3","h4","h5","h6","header","hgroup","html","i","iframe","img","input","ins","kbd","label","legend","li","main","mark","menu","nav","object","ol","p","q","quote","samp","section","span","strong","summary","sup","table","tbody","td","textarea","tfoot","th","thead","time","tr","ul","var","video"],t=["any-hover","any-pointer","aspect-ratio","color","color-gamut","color-index","device-aspect-ratio","device-height","device-width","display-mode","forced-colors","grid","height","hover","inverted-colors","monochrome","orientation","overflow-block","overflow-inline","pointer","prefers-color-scheme","prefers-contrast","prefers-reduced-motion","prefers-reduced-transparency","resolution","scan","scripting","update","width","min-width","max-width","min-height","max-height"],i=["active","any-link","blank","checked","current","default","defined","dir","disabled","drop","empty","enabled","first","first-child","first-of-type","fullscreen","future","focus","focus-visible","focus-within","has","host","host-context","hover","indeterminate","in-range","invalid","is","lang","last-child","last-of-type","left","link","local-link","not","nth-child","nth-col","nth-last-child","nth-last-col","nth-last-of-type","nth-of-type","only-child","only-of-type","optional","out-of-range","past","placeholder-shown","read-only","read-write","required","right","root","scope","target","target-within","user-invalid","valid","visited","where"],o=["after","backdrop","before","cue","cue-region","first-letter","first-line","grammar-error","marker","part","placeholder","selection","slotted","spelling-error"],r=["align-content","align-items","align-self","animation","animation-delay","animation-direction","animation-duration","animation-fill-mode","animation-iteration-count","animation-name","animation-play-state","animation-timing-function","auto","backface-visibility","background","background-attachment","background-clip","background-color","background-image","background-origin","background-position","background-repeat","background-size","border","border-bottom","border-bottom-color","border-bottom-left-radius","border-bottom-right-radius","border-bottom-style","border-bottom-width","border-collapse","border-color","border-image","border-image-outset","border-image-repeat","border-image-slice","border-image-source","border-image-width","border-left","border-left-color","border-left-style","border-left-width","border-radius","border-right","border-right-color","border-right-style","border-right-width","border-spacing","border-style","border-top","border-top-color","border-top-left-radius","border-top-right-radius","border-top-style","border-top-width","border-width","bottom","box-decoration-break","box-shadow","box-sizing","break-after","break-before","break-inside","caption-side","clear","clip","clip-path","color","column-count","column-fill","column-gap","column-rule","column-rule-color","column-rule-style","column-rule-width","column-span","column-width","columns","content","counter-increment","counter-reset","cursor","direction","display","empty-cells","filter","flex","flex-basis","flex-direction","flex-flow","flex-grow","flex-shrink","flex-wrap","float","font","font-display","font-family","font-feature-settings","font-kerning","font-language-override","font-size","font-size-adjust","font-smoothing","font-stretch","font-style","font-variant","font-variant-ligatures","font-variation-settings","font-weight","height","hyphens","icon","image-orientation","image-rendering","image-resolution","ime-mode","inherit","initial","justify-content","left","letter-spacing","line-height","list-style","list-style-image","list-style-position","list-style-type","margin","margin-bottom","margin-left","margin-right","margin-top","marks","mask","max-height","max-width","min-height","min-width","nav-down","nav-index","nav-left","nav-right","nav-up","none","normal","object-fit","object-position","opacity","order","orphans","outline","outline-color","outline-offset","outline-style","outline-width","overflow","overflow-wrap","overflow-x","overflow-y","padding","padding-bottom","padding-left","padding-right","padding-top","page-break-after","page-break-before","page-break-inside","perspective","perspective-origin","pointer-events","position","quotes","resize","right","src","tab-size","table-layout","text-align","text-align-last","text-decoration","text-decoration-color","text-decoration-line","text-decoration-style","text-indent","text-overflow","text-rendering","text-shadow","text-transform","text-underline-position","top","transform","transform-origin","transform-style","transition","transition-delay","transition-duration","transition-property","transition-timing-function","unicode-bidi","vertical-align","visibility","white-space","widows","width","word-break","word-spacing","word-wrap","z-index"].reverse() +;return n=>{const a=(e=>({IMPORTANT:{className:"meta",begin:"!important"}, +HEXCOLOR:{className:"number",begin:"#([a-fA-F0-9]{6}|[a-fA-F0-9]{3})"}, +ATTRIBUTE_SELECTOR_MODE:{className:"selector-attr",begin:/\[/,end:/\]/, +illegal:"$",contains:[e.APOS_STRING_MODE,e.QUOTE_STRING_MODE]} +}))(n),l=[n.APOS_STRING_MODE,n.QUOTE_STRING_MODE];return{name:"CSS", +case_insensitive:!0,illegal:/[=|'\$]/,keywords:{keyframePosition:"from to"}, +classNameAliases:{keyframePosition:"selector-tag"}, +contains:[n.C_BLOCK_COMMENT_MODE,{begin:/-(webkit|moz|ms|o)-(?=[a-z])/ +},n.CSS_NUMBER_MODE,{className:"selector-id",begin:/#[A-Za-z0-9_-]+/,relevance:0 +},{className:"selector-class",begin:"\\.[a-zA-Z-][a-zA-Z0-9_-]*",relevance:0 +},a.ATTRIBUTE_SELECTOR_MODE,{className:"selector-pseudo",variants:[{ +begin:":("+i.join("|")+")"},{begin:"::("+o.join("|")+")"}]},{ +className:"attribute",begin:"\\b("+r.join("|")+")\\b"},{begin:":",end:"[;}]", +contains:[a.HEXCOLOR,a.IMPORTANT,n.CSS_NUMBER_MODE,...l,{ +begin:/(url|data-uri)\(/,end:/\)/,relevance:0,keywords:{built_in:"url data-uri" +},contains:[{className:"string",begin:/[^)]/,endsWithParent:!0,excludeEnd:!0}] +},{className:"built_in",begin:/[\w-]+(?=\()/}]},{ +begin:(s=/@/,((...e)=>e.map((e=>(e=>e?"string"==typeof e?e:e.source:null)(e))).join(""))("(?=",s,")")), +end:"[{;]",relevance:0,illegal:/:/,contains:[{className:"keyword", +begin:/@-?\w[\w]*(-\w+)*/},{begin:/\s/,endsWithParent:!0,excludeEnd:!0, +relevance:0,keywords:{$pattern:/[a-z-]+/,keyword:"and or not only", +attribute:t.join(" ")},contains:[{begin:/[a-z-]+(?=:)/,className:"attribute" +},...l,n.CSS_NUMBER_MODE]}]},{className:"selector-tag", +begin:"\\b("+e.join("|")+")\\b"}]};var s}})());hljs.registerLanguage("apache",(()=>{"use strict";return e=>{const n={ +className:"number",begin:/\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}(:\d{1,5})?/} +;return{name:"Apache config",aliases:["apacheconf"],case_insensitive:!0, +contains:[e.HASH_COMMENT_MODE,{className:"section",begin:/<\/?/,end:/>/, +contains:[n,{className:"number",begin:/:\d{1,5}/ +},e.inherit(e.QUOTE_STRING_MODE,{relevance:0})]},{className:"attribute", +begin:/\w+/,relevance:0,keywords:{ +nomarkup:"order deny allow setenv rewriterule rewriteengine rewritecond documentroot sethandler errordocument loadmodule options header listen serverroot servername" +},starts:{end:/$/,relevance:0,keywords:{literal:"on off all deny allow"}, +contains:[{className:"meta",begin:/\s\[/,end:/\]$/},{className:"variable", +begin:/[\$%]\{/,end:/\}/,contains:["self",{className:"number",begin:/[$%]\d+/}] +},n,{className:"number",begin:/\d+/},e.QUOTE_STRING_MODE]}}],illegal:/\S/}} +})());hljs.registerLanguage("plaintext",(()=>{"use strict";return t=>({ +name:"Plain text",aliases:["text","txt"],disableAutodetect:!0})})());hljs.registerLanguage("cpp",(()=>{"use strict";function e(e){ +return t("(",e,")?")}function t(...e){return e.map((e=>{ +return(t=e)?"string"==typeof t?t:t.source:null;var t})).join("")}return n=>{ +const r=n.COMMENT("//","$",{contains:[{begin:/\\\n/}] +}),a="[a-zA-Z_]\\w*::",i="(decltype\\(auto\\)|"+e(a)+"[a-zA-Z_]\\w*"+e("<[^<>]+>")+")",s={ +className:"keyword",begin:"\\b[a-z\\d_]*_t\\b"},c={className:"string", +variants:[{begin:'(u8?|U|L)?"',end:'"',illegal:"\\n", +contains:[n.BACKSLASH_ESCAPE]},{ +begin:"(u8?|U|L)?'(\\\\(x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4,8}|[0-7]{3}|\\S)|.)", +end:"'",illegal:"."},n.END_SAME_AS_BEGIN({ +begin:/(?:u8?|U|L)?R"([^()\\ ]{0,16})\(/,end:/\)([^()\\ ]{0,16})"/})]},o={ +className:"number",variants:[{begin:"\\b(0b[01']+)"},{ +begin:"(-?)\\b([\\d']+(\\.[\\d']*)?|\\.[\\d']+)((ll|LL|l|L)(u|U)?|(u|U)(ll|LL|l|L)?|f|F|b|B)" +},{ +begin:"(-?)(\\b0[xX][a-fA-F0-9']+|(\\b[\\d']+(\\.[\\d']*)?|\\.[\\d']+)([eE][-+]?[\\d']+)?)" +}],relevance:0},l={className:"meta",begin:/#\s*[a-z]+\b/,end:/$/,keywords:{ +"meta-keyword":"if else elif endif define undef warning error line pragma _Pragma ifdef ifndef include" +},contains:[{begin:/\\\n/,relevance:0},n.inherit(c,{className:"meta-string"}),{ +className:"meta-string",begin:/<.*?>/},r,n.C_BLOCK_COMMENT_MODE]},d={ +className:"title",begin:e(a)+n.IDENT_RE,relevance:0 +},u=e(a)+n.IDENT_RE+"\\s*\\(",m={ +keyword:"int float while private char char8_t char16_t char32_t catch import module export virtual operator sizeof dynamic_cast|10 typedef const_cast|10 const for static_cast|10 union namespace unsigned long volatile static protected bool template mutable if public friend do goto auto void enum else break extern using asm case typeid wchar_t short reinterpret_cast|10 default double register explicit signed typename try this switch continue inline delete alignas alignof constexpr consteval constinit decltype concept co_await co_return co_yield requires noexcept static_assert thread_local restrict final override atomic_bool atomic_char atomic_schar atomic_uchar atomic_short atomic_ushort atomic_int atomic_uint atomic_long atomic_ulong atomic_llong atomic_ullong new throw return and and_eq bitand bitor compl not not_eq or or_eq xor xor_eq", +built_in:"_Bool _Complex _Imaginary", +_relevance_hints:["asin","atan2","atan","calloc","ceil","cosh","cos","exit","exp","fabs","floor","fmod","fprintf","fputs","free","frexp","auto_ptr","deque","list","queue","stack","vector","map","set","pair","bitset","multiset","multimap","unordered_set","fscanf","future","isalnum","isalpha","iscntrl","isdigit","isgraph","islower","isprint","ispunct","isspace","isupper","isxdigit","tolower","toupper","labs","ldexp","log10","log","malloc","realloc","memchr","memcmp","memcpy","memset","modf","pow","printf","putchar","puts","scanf","sinh","sin","snprintf","sprintf","sqrt","sscanf","strcat","strchr","strcmp","strcpy","strcspn","strlen","strncat","strncmp","strncpy","strpbrk","strrchr","strspn","strstr","tanh","tan","unordered_map","unordered_multiset","unordered_multimap","priority_queue","make_pair","array","shared_ptr","abort","terminate","abs","acos","vfprintf","vprintf","vsprintf","endl","initializer_list","unique_ptr","complex","imaginary","std","string","wstring","cin","cout","cerr","clog","stdin","stdout","stderr","stringstream","istringstream","ostringstream"], +literal:"true false nullptr NULL"},p={className:"function.dispatch",relevance:0, +keywords:m, +begin:t(/\b/,/(?!decltype)/,/(?!if)/,/(?!for)/,/(?!while)/,n.IDENT_RE,(_=/\s*\(/, +t("(?=",_,")")))};var _;const g=[p,l,s,r,n.C_BLOCK_COMMENT_MODE,o,c],b={ +variants:[{begin:/=/,end:/;/},{begin:/\(/,end:/\)/},{ +beginKeywords:"new throw return else",end:/;/}],keywords:m,contains:g.concat([{ +begin:/\(/,end:/\)/,keywords:m,contains:g.concat(["self"]),relevance:0}]), +relevance:0},f={className:"function",begin:"("+i+"[\\*&\\s]+)+"+u, +returnBegin:!0,end:/[{;=]/,excludeEnd:!0,keywords:m,illegal:/[^\w\s\*&:<>.]/, +contains:[{begin:"decltype\\(auto\\)",keywords:m,relevance:0},{begin:u, +returnBegin:!0,contains:[d],relevance:0},{begin:/::/,relevance:0},{begin:/:/, +endsWithParent:!0,contains:[c,o]},{className:"params",begin:/\(/,end:/\)/, +keywords:m,relevance:0,contains:[r,n.C_BLOCK_COMMENT_MODE,c,o,s,{begin:/\(/, +end:/\)/,keywords:m,relevance:0,contains:["self",r,n.C_BLOCK_COMMENT_MODE,c,o,s] +}]},s,r,n.C_BLOCK_COMMENT_MODE,l]};return{name:"C++", +aliases:["cc","c++","h++","hpp","hh","hxx","cxx"],keywords:m,illegal:"",keywords:m,contains:["self",s]},{begin:n.IDENT_RE+"::",keywords:m},{ +className:"class",beginKeywords:"enum class struct union",end:/[{;:<>=]/, +contains:[{beginKeywords:"final class struct"},n.TITLE_MODE]}]),exports:{ +preprocessor:l,strings:c,keywords:m}}}})());hljs.registerLanguage("http",(()=>{"use strict";function e(...e){ +return e.map((e=>{return(n=e)?"string"==typeof n?n:n.source:null;var n +})).join("")}return n=>{const a="HTTP/(2|1\\.[01])",s={className:"attribute", +begin:e("^",/[A-Za-z][A-Za-z0-9-]*/,"(?=\\:\\s)"),starts:{contains:[{ +className:"punctuation",begin:/: /,relevance:0,starts:{end:"$",relevance:0}}]} +},t=[s,{begin:"\\n\\n",starts:{subLanguage:[],endsWithParent:!0}}];return{ +name:"HTTP",aliases:["https"],illegal:/\S/,contains:[{begin:"^(?="+a+" \\d{3})", +end:/$/,contains:[{className:"meta",begin:a},{className:"number", +begin:"\\b\\d{3}\\b"}],starts:{end:/\b\B/,illegal:/\S/,contains:t}},{ +begin:"(?=^[A-Z]+ (.*?) "+a+"$)",end:/$/,contains:[{className:"string", +begin:" ",end:" ",excludeBegin:!0,excludeEnd:!0},{className:"meta",begin:a},{ +className:"keyword",begin:"[A-Z]+"}],starts:{end:/\b\B/,illegal:/\S/,contains:t} +},n.inherit(s,{relevance:0})]}}})());hljs.registerLanguage("properties",(()=>{"use strict";return e=>{ +var n="[ \\t\\f]*",a=n+"[:=]"+n,t="("+a+"|[ \\t\\f]+)",r="([^\\\\\\W:= \\t\\f\\n]|\\\\.)+",s="([^\\\\:= \\t\\f\\n]|\\\\.)+",i={ +end:t,relevance:0,starts:{className:"string",end:/$/,relevance:0,contains:[{ +begin:"\\\\\\\\"},{begin:"\\\\\\n"}]}};return{name:".properties", +case_insensitive:!0,illegal:/\S/,contains:[e.COMMENT("^\\s*[!#]","$"),{ +returnBegin:!0,variants:[{begin:r+a,relevance:1},{begin:r+"[ \\t\\f]+", +relevance:0}],contains:[{className:"attr",begin:r,endsParent:!0,relevance:0}], +starts:i},{begin:s+t,returnBegin:!0,relevance:0,contains:[{className:"meta", +begin:s,endsParent:!0,relevance:0}],starts:i},{className:"attr",relevance:0, +begin:s+n+"$"}]}}})());hljs.registerLanguage("makefile",(()=>{"use strict";return e=>{const i={ +className:"variable",variants:[{begin:"\\$\\("+e.UNDERSCORE_IDENT_RE+"\\)", +contains:[e.BACKSLASH_ESCAPE]},{begin:/\$[@%{"use strict";return e=>{const n={ +$pattern:/[A-Za-z]\w+|__\w+__/, +keyword:["and","as","assert","async","await","break","class","continue","def","del","elif","else","except","finally","for","from","global","if","import","in","is","lambda","nonlocal|10","not","or","pass","raise","return","try","while","with","yield"], +built_in:["__import__","abs","all","any","ascii","bin","bool","breakpoint","bytearray","bytes","callable","chr","classmethod","compile","complex","delattr","dict","dir","divmod","enumerate","eval","exec","filter","float","format","frozenset","getattr","globals","hasattr","hash","help","hex","id","input","int","isinstance","issubclass","iter","len","list","locals","map","max","memoryview","min","next","object","oct","open","ord","pow","print","property","range","repr","reversed","round","set","setattr","slice","sorted","staticmethod","str","sum","super","tuple","type","vars","zip"], +literal:["__debug__","Ellipsis","False","None","NotImplemented","True"], +type:["Any","Callable","Coroutine","Dict","List","Literal","Generic","Optional","Sequence","Set","Tuple","Type","Union"] +},a={className:"meta",begin:/^(>>>|\.\.\.) /},i={className:"subst",begin:/\{/, +end:/\}/,keywords:n,illegal:/#/},s={begin:/\{\{/,relevance:0},t={ +className:"string",contains:[e.BACKSLASH_ESCAPE],variants:[{ +begin:/([uU]|[bB]|[rR]|[bB][rR]|[rR][bB])?'''/,end:/'''/, +contains:[e.BACKSLASH_ESCAPE,a],relevance:10},{ +begin:/([uU]|[bB]|[rR]|[bB][rR]|[rR][bB])?"""/,end:/"""/, +contains:[e.BACKSLASH_ESCAPE,a],relevance:10},{ +begin:/([fF][rR]|[rR][fF]|[fF])'''/,end:/'''/, +contains:[e.BACKSLASH_ESCAPE,a,s,i]},{begin:/([fF][rR]|[rR][fF]|[fF])"""/, +end:/"""/,contains:[e.BACKSLASH_ESCAPE,a,s,i]},{begin:/([uU]|[rR])'/,end:/'/, +relevance:10},{begin:/([uU]|[rR])"/,end:/"/,relevance:10},{ +begin:/([bB]|[bB][rR]|[rR][bB])'/,end:/'/},{begin:/([bB]|[bB][rR]|[rR][bB])"/, +end:/"/},{begin:/([fF][rR]|[rR][fF]|[fF])'/,end:/'/, +contains:[e.BACKSLASH_ESCAPE,s,i]},{begin:/([fF][rR]|[rR][fF]|[fF])"/,end:/"/, +contains:[e.BACKSLASH_ESCAPE,s,i]},e.APOS_STRING_MODE,e.QUOTE_STRING_MODE] +},r="[0-9](_?[0-9])*",l=`(\\b(${r}))?\\.(${r})|\\b(${r})\\.`,b={ +className:"number",relevance:0,variants:[{ +begin:`(\\b(${r})|(${l}))[eE][+-]?(${r})[jJ]?\\b`},{begin:`(${l})[jJ]?`},{ +begin:"\\b([1-9](_?[0-9])*|0+(_?0)*)[lLjJ]?\\b"},{ +begin:"\\b0[bB](_?[01])+[lL]?\\b"},{begin:"\\b0[oO](_?[0-7])+[lL]?\\b"},{ +begin:"\\b0[xX](_?[0-9a-fA-F])+[lL]?\\b"},{begin:`\\b(${r})[jJ]\\b`}]},o={ +className:"comment", +begin:(d=/# type:/,((...e)=>e.map((e=>(e=>e?"string"==typeof e?e:e.source:null)(e))).join(""))("(?=",d,")")), +end:/$/,keywords:n,contains:[{begin:/# type:/},{begin:/#/,end:/\b\B/, +endsWithParent:!0}]},c={className:"params",variants:[{className:"", +begin:/\(\s*\)/,skip:!0},{begin:/\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0, +keywords:n,contains:["self",a,b,t,e.HASH_COMMENT_MODE]}]};var d +;return i.contains=[t,b,a],{name:"Python",aliases:["py","gyp","ipython"], +keywords:n,illegal:/(<\/|->|\?)|=>/,contains:[a,b,{begin:/\bself\b/},{ +beginKeywords:"if",relevance:0},t,o,e.HASH_COMMENT_MODE,{variants:[{ +className:"function",beginKeywords:"def"},{className:"class", +beginKeywords:"class"}],end:/:/,illegal:/[${=;\n,]/, +contains:[e.UNDERSCORE_TITLE_MODE,c,{begin:/->/,endsWithParent:!0,keywords:n}] +},{className:"meta",begin:/^[\t ]*@/,end:/(?=#)|$/,contains:[b,c,t]}]}}})());hljs.registerLanguage("python-repl",(()=>{"use strict";return s=>({ +aliases:["pycon"],contains:[{className:"meta",starts:{end:/ |$/,starts:{end:"$", +subLanguage:"python"}},variants:[{begin:/^>>>(?=[ ]|$)/},{ +begin:/^\.\.\.(?=[ ]|$)/}]}]})})());hljs.registerLanguage("protobuf",(()=>{"use strict";return e=>({ +name:"Protocol Buffers",keywords:{ +keyword:"package import option optional required repeated group oneof", +built_in:"double float int32 int64 uint32 uint64 sint32 sint64 fixed32 fixed64 sfixed32 sfixed64 bool string bytes", +literal:"true false"}, +contains:[e.QUOTE_STRING_MODE,e.NUMBER_MODE,e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,{ +className:"class",beginKeywords:"message enum service",end:/\{/,illegal:/\n/, +contains:[e.inherit(e.TITLE_MODE,{starts:{endsWithParent:!0,excludeEnd:!0}})]},{ +className:"function",beginKeywords:"rpc",end:/[{;]/,excludeEnd:!0, +keywords:"rpc returns"},{begin:/^\s*[A-Z_]+(?=\s*=[^\n]+;$)/}]})})());hljs.registerLanguage("java",(()=>{"use strict" +;var e="\\.([0-9](_*[0-9])*)",n="[0-9a-fA-F](_*[0-9a-fA-F])*",a={ +className:"number",variants:[{ +begin:`(\\b([0-9](_*[0-9])*)((${e})|\\.)?|(${e}))[eE][+-]?([0-9](_*[0-9])*)[fFdD]?\\b` +},{begin:`\\b([0-9](_*[0-9])*)((${e})[fFdD]?\\b|\\.([fFdD]\\b)?)`},{ +begin:`(${e})[fFdD]?\\b`},{begin:"\\b([0-9](_*[0-9])*)[fFdD]\\b"},{ +begin:`\\b0[xX]((${n})\\.?|(${n})?\\.(${n}))[pP][+-]?([0-9](_*[0-9])*)[fFdD]?\\b` +},{begin:"\\b(0|[1-9](_*[0-9])*)[lL]?\\b"},{begin:`\\b0[xX](${n})[lL]?\\b`},{ +begin:"\\b0(_*[0-7])*[lL]?\\b"},{begin:"\\b0[bB][01](_*[01])*[lL]?\\b"}], +relevance:0};return e=>{ +var n="false synchronized int abstract float private char boolean var static null if const for true while long strictfp finally protected import native final void enum else break transient catch instanceof byte super volatile case assert short package default double public try this switch continue throws protected public private module requires exports do",s={ +className:"meta",begin:"@[\xc0-\u02b8a-zA-Z_$][\xc0-\u02b8a-zA-Z_$0-9]*", +contains:[{begin:/\(/,end:/\)/,contains:["self"]}]};const r=a;return{ +name:"Java",aliases:["jsp"],keywords:n,illegal:/<\/|#/, +contains:[e.COMMENT("/\\*\\*","\\*/",{relevance:0,contains:[{begin:/\w+@/, +relevance:0},{className:"doctag",begin:"@[A-Za-z]+"}]}),{ +begin:/import java\.[a-z]+\./,keywords:"import",relevance:2 +},e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,{ +className:"class",beginKeywords:"class interface enum",end:/[{;=]/, +excludeEnd:!0,relevance:1,keywords:"class interface enum",illegal:/[:"\[\]]/, +contains:[{beginKeywords:"extends implements"},e.UNDERSCORE_TITLE_MODE]},{ +beginKeywords:"new throw return else",relevance:0},{className:"class", +begin:"record\\s+"+e.UNDERSCORE_IDENT_RE+"\\s*\\(",returnBegin:!0,excludeEnd:!0, +end:/[{;=]/,keywords:n,contains:[{beginKeywords:"record"},{ +begin:e.UNDERSCORE_IDENT_RE+"\\s*\\(",returnBegin:!0,relevance:0, +contains:[e.UNDERSCORE_TITLE_MODE]},{className:"params",begin:/\(/,end:/\)/, +keywords:n,relevance:0,contains:[e.C_BLOCK_COMMENT_MODE] +},e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]},{className:"function", +begin:"([\xc0-\u02b8a-zA-Z_$][\xc0-\u02b8a-zA-Z_$0-9]*(<[\xc0-\u02b8a-zA-Z_$][\xc0-\u02b8a-zA-Z_$0-9]*(\\s*,\\s*[\xc0-\u02b8a-zA-Z_$][\xc0-\u02b8a-zA-Z_$0-9]*)*>)?\\s+)+"+e.UNDERSCORE_IDENT_RE+"\\s*\\(", +returnBegin:!0,end:/[{;=]/,excludeEnd:!0,keywords:n,contains:[{ +begin:e.UNDERSCORE_IDENT_RE+"\\s*\\(",returnBegin:!0,relevance:0, +contains:[e.UNDERSCORE_TITLE_MODE]},{className:"params",begin:/\(/,end:/\)/, +keywords:n,relevance:0, +contains:[s,e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,r,e.C_BLOCK_COMMENT_MODE] +},e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]},r,s]}}})());hljs.registerLanguage("coffeescript",(()=>{"use strict" +;const e=["as","in","of","if","for","while","finally","var","new","function","do","return","void","else","break","catch","instanceof","with","throw","case","default","try","switch","continue","typeof","delete","let","yield","const","class","debugger","async","await","static","import","from","export","extends"],n=["true","false","null","undefined","NaN","Infinity"],a=[].concat(["setInterval","setTimeout","clearInterval","clearTimeout","require","exports","eval","isFinite","isNaN","parseFloat","parseInt","decodeURI","decodeURIComponent","encodeURI","encodeURIComponent","escape","unescape"],["arguments","this","super","console","window","document","localStorage","module","global"],["Intl","DataView","Number","Math","Date","String","RegExp","Object","Function","Boolean","Error","Symbol","Set","Map","WeakSet","WeakMap","Proxy","Reflect","JSON","Promise","Float64Array","Int16Array","Int32Array","Int8Array","Uint16Array","Uint32Array","Float32Array","Array","Uint8Array","Uint8ClampedArray","ArrayBuffer","BigInt64Array","BigUint64Array","BigInt"],["EvalError","InternalError","RangeError","ReferenceError","SyntaxError","TypeError","URIError"]) +;return r=>{const t={ +keyword:e.concat(["then","unless","until","loop","by","when","and","or","is","isnt","not"]).filter((i=["var","const","let","function","static"], +e=>!i.includes(e))),literal:n.concat(["yes","no","on","off"]), +built_in:a.concat(["npm","print"])};var i;const s="[A-Za-z$_][0-9A-Za-z$_]*",o={ +className:"subst",begin:/#\{/,end:/\}/,keywords:t +},c=[r.BINARY_NUMBER_MODE,r.inherit(r.C_NUMBER_MODE,{starts:{end:"(\\s*/)?", +relevance:0}}),{className:"string",variants:[{begin:/'''/,end:/'''/, +contains:[r.BACKSLASH_ESCAPE]},{begin:/'/,end:/'/,contains:[r.BACKSLASH_ESCAPE] +},{begin:/"""/,end:/"""/,contains:[r.BACKSLASH_ESCAPE,o]},{begin:/"/,end:/"/, +contains:[r.BACKSLASH_ESCAPE,o]}]},{className:"regexp",variants:[{begin:"///", +end:"///",contains:[o,r.HASH_COMMENT_MODE]},{begin:"//[gim]{0,3}(?=\\W)", +relevance:0},{begin:/\/(?![ *]).*?(?![\\]).\/[gim]{0,3}(?=\W)/}]},{begin:"@"+s +},{subLanguage:"javascript",excludeBegin:!0,excludeEnd:!0,variants:[{ +begin:"```",end:"```"},{begin:"`",end:"`"}]}];o.contains=c +;const l=r.inherit(r.TITLE_MODE,{begin:s}),d="(\\(.*\\)\\s*)?\\B[-=]>",g={ +className:"params",begin:"\\([^\\(]",returnBegin:!0,contains:[{begin:/\(/, +end:/\)/,keywords:t,contains:["self"].concat(c)}]};return{name:"CoffeeScript", +aliases:["coffee","cson","iced"],keywords:t,illegal:/\/\*/, +contains:c.concat([r.COMMENT("###","###"),r.HASH_COMMENT_MODE,{ +className:"function",begin:"^\\s*"+s+"\\s*=\\s*"+d,end:"[-=]>",returnBegin:!0, +contains:[l,g]},{begin:/[:\(,=]\s*/,relevance:0,contains:[{className:"function", +begin:d,end:"[-=]>",returnBegin:!0,contains:[g]}]},{className:"class", +beginKeywords:"class",end:"$",illegal:/[:="\[\]]/,contains:[{ +beginKeywords:"extends",endsWithParent:!0,illegal:/[:="\[\]]/,contains:[l]},l] +},{begin:s+":",end:":",returnBegin:!0,returnEnd:!0,relevance:0}])}}})());hljs.registerLanguage("objectivec",(()=>{"use strict";return e=>{ +const n=/[a-zA-Z@][a-zA-Z0-9_]*/,_={$pattern:n, +keyword:"@interface @class @protocol @implementation"};return{ +name:"Objective-C",aliases:["mm","objc","obj-c","obj-c++","objective-c++"], +keywords:{$pattern:n, +keyword:"int float while char export sizeof typedef const struct for union unsigned long volatile static bool mutable if do return goto void enum else break extern asm case short default double register explicit signed typename this switch continue wchar_t inline readonly assign readwrite self @synchronized id typeof nonatomic super unichar IBOutlet IBAction strong weak copy in out inout bycopy byref oneway __strong __weak __block __autoreleasing @private @protected @public @try @property @end @throw @catch @finally @autoreleasepool @synthesize @dynamic @selector @optional @required @encode @package @import @defs @compatibility_alias __bridge __bridge_transfer __bridge_retained __bridge_retain __covariant __contravariant __kindof _Nonnull _Nullable _Null_unspecified __FUNCTION__ __PRETTY_FUNCTION__ __attribute__ getter setter retain unsafe_unretained nonnull nullable null_unspecified null_resettable class instancetype NS_DESIGNATED_INITIALIZER NS_UNAVAILABLE NS_REQUIRES_SUPER NS_RETURNS_INNER_POINTER NS_INLINE NS_AVAILABLE NS_DEPRECATED NS_ENUM NS_OPTIONS NS_SWIFT_UNAVAILABLE NS_ASSUME_NONNULL_BEGIN NS_ASSUME_NONNULL_END NS_REFINED_FOR_SWIFT NS_SWIFT_NAME NS_SWIFT_NOTHROW NS_DURING NS_HANDLER NS_ENDHANDLER NS_VALUERETURN NS_VOIDRETURN", +literal:"false true FALSE TRUE nil YES NO NULL", +built_in:"BOOL dispatch_once_t dispatch_queue_t dispatch_sync dispatch_async dispatch_once" +},illegal:"/,end:/$/, +illegal:"\\n"},e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]},{ +className:"class",begin:"("+_.keyword.split(" ").join("|")+")\\b",end:/(\{|$)/, +excludeEnd:!0,keywords:_,contains:[e.UNDERSCORE_TITLE_MODE]},{ +begin:"\\."+e.UNDERSCORE_IDENT_RE,relevance:0}]}}})());hljs.registerLanguage("javascript",(()=>{"use strict" +;const e="[A-Za-z$_][0-9A-Za-z$_]*",n=["as","in","of","if","for","while","finally","var","new","function","do","return","void","else","break","catch","instanceof","with","throw","case","default","try","switch","continue","typeof","delete","let","yield","const","class","debugger","async","await","static","import","from","export","extends"],a=["true","false","null","undefined","NaN","Infinity"],s=[].concat(["setInterval","setTimeout","clearInterval","clearTimeout","require","exports","eval","isFinite","isNaN","parseFloat","parseInt","decodeURI","decodeURIComponent","encodeURI","encodeURIComponent","escape","unescape"],["arguments","this","super","console","window","document","localStorage","module","global"],["Intl","DataView","Number","Math","Date","String","RegExp","Object","Function","Boolean","Error","Symbol","Set","Map","WeakSet","WeakMap","Proxy","Reflect","JSON","Promise","Float64Array","Int16Array","Int32Array","Int8Array","Uint16Array","Uint32Array","Float32Array","Array","Uint8Array","Uint8ClampedArray","ArrayBuffer","BigInt64Array","BigUint64Array","BigInt"],["EvalError","InternalError","RangeError","ReferenceError","SyntaxError","TypeError","URIError"]) +;function r(e){return t("(?=",e,")")}function t(...e){return e.map((e=>{ +return(n=e)?"string"==typeof n?n:n.source:null;var n})).join("")}return i=>{ +const c=e,o={begin:/<[A-Za-z0-9\\._:-]+/,end:/\/[A-Za-z0-9\\._:-]+>|\/>/, +isTrulyOpeningTag:(e,n)=>{const a=e[0].length+e.index,s=e.input[a] +;"<"!==s?">"===s&&(((e,{after:n})=>{const a="", +returnBegin:!0,end:"\\s*=>",contains:[{className:"params",variants:[{ +begin:i.UNDERSCORE_IDENT_RE,relevance:0},{className:null,begin:/\(\s*\)/,skip:!0 +},{begin:/\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0,keywords:l,contains:f}]}] +},{begin:/,/,relevance:0},{className:"",begin:/\s/,end:/\s*/,skip:!0},{ +variants:[{begin:"<>",end:""},{begin:o.begin,"on:begin":o.isTrulyOpeningTag, +end:o.end}],subLanguage:"xml",contains:[{begin:o.begin,end:o.end,skip:!0, +contains:["self"]}]}],relevance:0},{className:"function", +beginKeywords:"function",end:/[{;]/,excludeEnd:!0,keywords:l, +contains:["self",i.inherit(i.TITLE_MODE,{begin:c}),p],illegal:/%/},{ +beginKeywords:"while if switch catch for"},{className:"function", +begin:i.UNDERSCORE_IDENT_RE+"\\([^()]*(\\([^()]*(\\([^()]*\\)[^()]*)*\\)[^()]*)*\\)\\s*\\{", +returnBegin:!0,contains:[p,i.inherit(i.TITLE_MODE,{begin:c})]},{variants:[{ +begin:"\\."+c},{begin:"\\$"+c}],relevance:0},{className:"class", +beginKeywords:"class",end:/[{;=]/,excludeEnd:!0,illegal:/[:"[\]]/,contains:[{ +beginKeywords:"extends"},i.UNDERSCORE_TITLE_MODE]},{begin:/\b(?=constructor)/, +end:/[{;]/,excludeEnd:!0,contains:[i.inherit(i.TITLE_MODE,{begin:c}),"self",p] +},{begin:"(get|set)\\s+(?="+c+"\\()",end:/\{/,keywords:"get set", +contains:[i.inherit(i.TITLE_MODE,{begin:c}),{begin:/\(\)/},p]},{begin:/\$[(.]/}] +}}})());hljs.registerLanguage("c",(()=>{"use strict";function e(e){ +return((...e)=>e.map((e=>(e=>e?"string"==typeof e?e:e.source:null)(e))).join(""))("(",e,")?") +}return t=>{const n=t.COMMENT("//","$",{contains:[{begin:/\\\n/}] +}),r="[a-zA-Z_]\\w*::",a="(decltype\\(auto\\)|"+e(r)+"[a-zA-Z_]\\w*"+e("<[^<>]+>")+")",i={ +className:"keyword",begin:"\\b[a-z\\d_]*_t\\b"},s={className:"string", +variants:[{begin:'(u8?|U|L)?"',end:'"',illegal:"\\n", +contains:[t.BACKSLASH_ESCAPE]},{ +begin:"(u8?|U|L)?'(\\\\(x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4,8}|[0-7]{3}|\\S)|.)", +end:"'",illegal:"."},t.END_SAME_AS_BEGIN({ +begin:/(?:u8?|U|L)?R"([^()\\ ]{0,16})\(/,end:/\)([^()\\ ]{0,16})"/})]},o={ +className:"number",variants:[{begin:"\\b(0b[01']+)"},{ +begin:"(-?)\\b([\\d']+(\\.[\\d']*)?|\\.[\\d']+)((ll|LL|l|L)(u|U)?|(u|U)(ll|LL|l|L)?|f|F|b|B)" +},{ +begin:"(-?)(\\b0[xX][a-fA-F0-9']+|(\\b[\\d']+(\\.[\\d']*)?|\\.[\\d']+)([eE][-+]?[\\d']+)?)" +}],relevance:0},c={className:"meta",begin:/#\s*[a-z]+\b/,end:/$/,keywords:{ +"meta-keyword":"if else elif endif define undef warning error line pragma _Pragma ifdef ifndef include" +},contains:[{begin:/\\\n/,relevance:0},t.inherit(s,{className:"meta-string"}),{ +className:"meta-string",begin:/<.*?>/},n,t.C_BLOCK_COMMENT_MODE]},l={ +className:"title",begin:e(r)+t.IDENT_RE,relevance:0 +},d=e(r)+t.IDENT_RE+"\\s*\\(",u={ +keyword:"int float while private char char8_t char16_t char32_t catch import module export virtual operator sizeof dynamic_cast|10 typedef const_cast|10 const for static_cast|10 union namespace unsigned long volatile static protected bool template mutable if public friend do goto auto void enum else break extern using asm case typeid wchar_t short reinterpret_cast|10 default double register explicit signed typename try this switch continue inline delete alignas alignof constexpr consteval constinit decltype concept co_await co_return co_yield requires noexcept static_assert thread_local restrict final override atomic_bool atomic_char atomic_schar atomic_uchar atomic_short atomic_ushort atomic_int atomic_uint atomic_long atomic_ulong atomic_llong atomic_ullong new throw return and and_eq bitand bitor compl not not_eq or or_eq xor xor_eq", +built_in:"std string wstring cin cout cerr clog stdin stdout stderr stringstream istringstream ostringstream auto_ptr deque list queue stack vector map set pair bitset multiset multimap unordered_set unordered_map unordered_multiset unordered_multimap priority_queue make_pair array shared_ptr abort terminate abs acos asin atan2 atan calloc ceil cosh cos exit exp fabs floor fmod fprintf fputs free frexp fscanf future isalnum isalpha iscntrl isdigit isgraph islower isprint ispunct isspace isupper isxdigit tolower toupper labs ldexp log10 log malloc realloc memchr memcmp memcpy memset modf pow printf putchar puts scanf sinh sin snprintf sprintf sqrt sscanf strcat strchr strcmp strcpy strcspn strlen strncat strncmp strncpy strpbrk strrchr strspn strstr tanh tan vfprintf vprintf vsprintf endl initializer_list unique_ptr _Bool complex _Complex imaginary _Imaginary", +literal:"true false nullptr NULL"},m=[c,i,n,t.C_BLOCK_COMMENT_MODE,o,s],p={ +variants:[{begin:/=/,end:/;/},{begin:/\(/,end:/\)/},{ +beginKeywords:"new throw return else",end:/;/}],keywords:u,contains:m.concat([{ +begin:/\(/,end:/\)/,keywords:u,contains:m.concat(["self"]),relevance:0}]), +relevance:0},_={className:"function",begin:"("+a+"[\\*&\\s]+)+"+d, +returnBegin:!0,end:/[{;=]/,excludeEnd:!0,keywords:u,illegal:/[^\w\s\*&:<>.]/, +contains:[{begin:"decltype\\(auto\\)",keywords:u,relevance:0},{begin:d, +returnBegin:!0,contains:[l],relevance:0},{className:"params",begin:/\(/, +end:/\)/,keywords:u,relevance:0,contains:[n,t.C_BLOCK_COMMENT_MODE,s,o,i,{ +begin:/\(/,end:/\)/,keywords:u,relevance:0, +contains:["self",n,t.C_BLOCK_COMMENT_MODE,s,o,i]}] +},i,n,t.C_BLOCK_COMMENT_MODE,c]};return{name:"C",aliases:["h"],keywords:u, +disableAutodetect:!0,illegal:"",keywords:u,contains:["self",i]},{begin:t.IDENT_RE+"::",keywords:u},{ +className:"class",beginKeywords:"enum class struct union",end:/[{;:<>=]/, +contains:[{beginKeywords:"final class struct"},t.TITLE_MODE]}]),exports:{ +preprocessor:c,strings:s,keywords:u}}}})());hljs.registerLanguage("go",(()=>{"use strict";return e=>{const n={ +keyword:"break default func interface select case map struct chan else goto package switch const fallthrough if range type continue for import return var go defer bool byte complex64 complex128 float32 float64 int8 int16 int32 int64 string uint8 uint16 uint32 uint64 int uint uintptr rune", +literal:"true false iota nil", +built_in:"append cap close complex copy imag len make new panic print println real recover delete" +};return{name:"Go",aliases:["golang"],keywords:n,illegal:" Date: Tue, 27 Sep 2022 10:26:25 -0400 Subject: [PATCH 02/22] Fix markdown lint errors. (#798) See https://github.com/celestiaorg/celestia-app/actions/runs/3136139613/jobs/5092673489 --- specs/src/specs/data_structures.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/src/specs/data_structures.md b/specs/src/specs/data_structures.md index 486da11edf..aeb30b7be3 100644 --- a/specs/src/specs/data_structures.md +++ b/specs/src/specs/data_structures.md @@ -362,7 +362,7 @@ A compact commitment can be computed by taking the [hash](#hashing) of the [seri | `siblingMins` | [NamespaceID](#type-aliases)`[]` | Sibling min namespace IDs. | | `siblingMaxes` | [NamespaceID](#type-aliases)`[]` | Sibling max namespace IDs. | -When verifying an NMT proof, the root hash is checked by reconstructing the root node `root_node` with the computed `root_node.v` (computed as with a [plain Merkle proof](#binarymerkletreeproof)) and the provided `rootNamespaceIDMin` and `rootNamespaceIDMax` as the `root_node.n_min` and `root_node.n_max`, respectively. +When verifying an NMT proof, the root hash is checked by reconstructing the root node `root_node` with the computed `root_node.v` (computed as with a [plain Merkle proof](#binarymerkletreeinclusionproof)) and the provided `rootNamespaceIDMin` and `rootNamespaceIDMax` as the `root_node.n_min` and `root_node.n_max`, respectively. ### Sparse Merkle Tree @@ -412,7 +412,7 @@ node.v = h(0x01, l.v, r.v) #### SparseMerkleTreeInclusionProof -SMTs can further be extended with _compact_ proofs. [Merkle proofs](#verifying-annotated-merkle-proofs) are composed, among other things, of a list of sibling node values. We note that, since nodes that are roots of empty subtrees have known values (the default value), these values do not need to be provided explicitly; it is sufficient to simply identify which siblings in the Merkle branch are roots of empty subtrees, which can be done with one bit per sibling. +SMTs can further be extended with _compact_ proofs. Merkle proofs are composed, among other things, of a list of sibling node values. We note that, since nodes that are roots of empty subtrees have known values (the default value), these values do not need to be provided explicitly; it is sufficient to simply identify which siblings in the Merkle branch are roots of empty subtrees, which can be done with one bit per sibling. For a Merkle branch of height `h`, an `h`-bit value is appended to the proof. The lowest bit corresponds to the sibling of the leaf node, and each higher bit corresponds to the next parent. A value of `1` indicates that the next value in the list of values provided explicitly in the proof should be used, and a value of `0` indicates that the default value should be used. @@ -820,7 +820,7 @@ A number of subtrees are maintained: 1. [Active validator set](#validator) 1. [Inactive validator set](#validator) 1. [Delegation set](#delegation) -1. [Message shares paid for](#message-paid) +1. [Message shares paid for](#messagepaid) ### StateElement From dcf6d4ae5320cd76713334bf198a860255a626d2 Mon Sep 17 00:00:00 2001 From: John Adler Date: Tue, 27 Sep 2022 10:31:24 -0400 Subject: [PATCH 03/22] Add deploy step to mdbook workflow. (#797) --- .github/workflows/gh-pages.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 157c1c3077..f53a986d01 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -19,3 +19,11 @@ jobs: - name: Build book run: mdbook build specs + + - name: Deploy main + if: github.ref == 'refs/heads/main' && github.event_name == 'push' + uses: peaceiris/actions-gh-pages@v3 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: ./specs/book + destination_dir: main From dac49d17967626309d19a80f0249ddae71f0e920 Mon Sep 17 00:00:00 2001 From: John Adler Date: Tue, 27 Sep 2022 10:51:22 -0400 Subject: [PATCH 04/22] Run table formatter. (#799) Run [Markdown Table Prettifier](https://github.com/darkriszty/MarkdownTablePrettify-VSCodeExt). --- specs/src/specs/data_structures.md | 8 ++++---- specs/src/specs/networking.md | 32 +++++++++++++++--------------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/specs/src/specs/data_structures.md b/specs/src/specs/data_structures.md index aeb30b7be3..42a7d7f2c2 100644 --- a/specs/src/specs/data_structures.md +++ b/specs/src/specs/data_structures.md @@ -826,10 +826,10 @@ A number of subtrees are maintained: Data structure for state elements is given below: -| name | type | description | -|-------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------| -| `key` | `byte[32]` | Keys are byte arrays with size 32. | -| `value` | [Account](#account), [Delegation](#delegation), [Validator](#validator), [ActiveValidatorCount](#activevalidatorcount), [ActiveVotingPower](#activevotingpower), [ProposerBlockReward](#proposerblockreward), [ProposerInitialVotingPower](#proposerinitialvotingpower), [ValidatorQueueHead](#validatorqueuehead), [MessagePaidHead](#messagepaidhead) | `value` can be of different types depending on the state elements listed below. There exists a unique protobuf for different state elements. | +| name | type | description | +|---------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------| +| `key` | `byte[32]` | Keys are byte arrays with size 32. | +| `value` | [Account](#account), [Delegation](#delegation), [Validator](#validator), [ActiveValidatorCount](#activevalidatorcount), [ActiveVotingPower](#activevotingpower), [ProposerBlockReward](#proposerblockreward), [ProposerInitialVotingPower](#proposerinitialvotingpower), [ValidatorQueueHead](#validatorqueuehead), [MessagePaidHead](#messagepaidhead) | `value` can be of different types depending on the state elements listed below. There exists a unique protobuf for different state elements. | ### Account diff --git a/specs/src/specs/networking.md b/specs/src/specs/networking.md index 73edd84f48..e7b82c7120 100644 --- a/specs/src/specs/networking.md +++ b/specs/src/specs/networking.md @@ -78,8 +78,8 @@ If a malicious block producer incorrectly computes the 2D Reed-Solomon code for ### ShareProof -| name | type | description | -|------------|---------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------| +| name | type | description | +|------------|---------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------| | `share` | [Share](./data_structures.md#share) | The share. | | `proof` | [NamespaceMerkleTreeInclusionProof](./data_structures.md#namespacemerkletreeinclusionproof) | The Merkle proof of the share in the offending row or column root. | | `isCol` | `bool` | A Boolean indicating if the proof is from a row root or column root; `false` if it is a row root. | @@ -93,12 +93,12 @@ Defined as `BadEncodingFraudProof`: {{#include ./proto/types.proto:BadEncodingFraudProof}} ``` -| name | type | description | -|---------------|---------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------| -| `height` | [Height](./data_structures.md#type-aliases) | Height of the block with the offending row or column. | -| `shareProofs` | [ShareProof](#shareproof)`[]` | The available shares in the offending row or column. | -| `isCol` | `bool` | A Boolean indicating if it is an offending row or column; `false` if it is a row. | -| `position` | `uint64` | The index of the offending row or column in the square. | +| name | type | description | +|---------------|---------------------------------------------|-----------------------------------------------------------------------------------| +| `height` | [Height](./data_structures.md#type-aliases) | Height of the block with the offending row or column. | +| `shareProofs` | [ShareProof](#shareproof)`[]` | The available shares in the offending row or column. | +| `isCol` | `bool` | A Boolean indicating if it is an offending row or column; `false` if it is a row. | +| `position` | `uint64` | The index of the offending row or column in the square. | ## Invalid State Update @@ -112,11 +112,11 @@ Defined as `StateFraudProof`: {{#include ./proto/types.proto:StateFraudProof}} ``` -| name | type | description | -|----------------------------|------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `height` | [Height](./data_structures.md#type-aliases) | Height of the block with the intermediate state roots. Subtracting one from `height` gives the height of the block with the transactions. | -| `transactionShareProofs` | [ShareProof](#shareproof)`[]` | `isCol` of type `bool` must be `false`. | -| `isrShareProofs` | [ShareProof](#shareproof)`[]` | `isCol` of type `bool` must be `false`. | -| `index` | `uint64` | Index for connecting the [WrappedIntermediateStateRoot](./data_structures.md#wrappedintermediatestateroot) and [WrappedTransaction](./data_structures.md#wrappedtransaction) after shares are parsed. | -| `intermediateStateElements`| [StateElement](./data_structures.md#stateelement)`[]` | State elements that were changed by the transactions. | -| `stateInclusionProofs` | [SparseMerkleTreeInclusionProof](./data_structures.md#sparsemerkletreeinclusionproof)`[]`| SparseMerkleTree inclusion proofs for the state elements. | +| name | type | description | +|-----------------------------|-------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `height` | [Height](./data_structures.md#type-aliases) | Height of the block with the intermediate state roots. Subtracting one from `height` gives the height of the block with the transactions. | +| `transactionShareProofs` | [ShareProof](#shareproof)`[]` | `isCol` of type `bool` must be `false`. | +| `isrShareProofs` | [ShareProof](#shareproof)`[]` | `isCol` of type `bool` must be `false`. | +| `index` | `uint64` | Index for connecting the [WrappedIntermediateStateRoot](./data_structures.md#wrappedintermediatestateroot) and [WrappedTransaction](./data_structures.md#wrappedtransaction) after shares are parsed. | +| `intermediateStateElements` | [StateElement](./data_structures.md#stateelement)`[]` | State elements that were changed by the transactions. | +| `stateInclusionProofs` | [SparseMerkleTreeInclusionProof](./data_structures.md#sparsemerkletreeinclusionproof)`[]` | SparseMerkleTree inclusion proofs for the state elements. | From 60f341aa21f6661f013abbef8ec024b6b7958e59 Mon Sep 17 00:00:00 2001 From: John Adler Date: Tue, 27 Sep 2022 11:13:17 -0400 Subject: [PATCH 05/22] Add install prereq. (#800) Ref: https://github.com/celestiaorg/celestia-app/pull/793#discussion_r980600340 by @rootulp --- specs/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/specs/README.md b/specs/README.md index d9ef8443d2..e12959d126 100644 --- a/specs/README.md +++ b/specs/README.md @@ -2,6 +2,8 @@ ## Building From Source +Prerequisite: [install mdBook](https://rust-lang.github.io/mdBook/guide/installation.html). + To build book: ```sh From 2cc65da4e2663b4ba94e8be7861fe81f2be5d09c Mon Sep 17 00:00:00 2001 From: Rootul P Date: Thu, 29 Sep 2022 09:29:03 -0600 Subject: [PATCH 06/22] chore: specs-staging CODEOWNERS (#812) Closes https://github.com/celestiaorg/celestia-app/issues/807 --- .github/CODEOWNERS | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000000..7b71445279 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,17 @@ +# CODEOWNERS: + +# Everything goes through the following "global owners" by default. Unless a later +# match takes precedence, the "global owners" will be requested for review when +# someone opens a PR. Note that the last matching pattern takes precedence, so +# global owners are only requested if there isn't a more specific codeowner +# specified below. For this reason, the global owners are often repeated in +# directory-level definitions. + +# global owners +* @evan-forbes + +# directory owners +docs @liamsi @adlerjohn @MSevey +specs @liamsi @adlerjohn @MSevey +x/qgb @SweeXordious +pkg/shares @rootulp From eb3791e895315bca0e6100cf65eee71ffe7f6343 Mon Sep 17 00:00:00 2001 From: Rootul P Date: Wed, 19 Oct 2022 10:09:13 -0600 Subject: [PATCH 07/22] docs: describe message length varint (#806) Closes https://github.com/celestiaorg/celestia-app/issues/728 --- specs/src/specs/data_structures.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/src/specs/data_structures.md b/specs/src/specs/data_structures.md index 42a7d7f2c2..0d0dc682b0 100644 --- a/specs/src/specs/data_structures.md +++ b/specs/src/specs/data_structures.md @@ -495,7 +495,8 @@ For shares **with a reserved namespace ID through [`NAMESPACE_ID_MAX_RESERVED`]( For shares **with a namespace ID above [`NAMESPACE_ID_MAX_RESERVED`](./consensus.md#constants) but below [`PARITY_SHARE_NAMESPACE_ID`](./consensus.md#constants)**: - The first [`NAMESPACE_ID_BYTES`](./consensus.md#constants) of a share's raw data `rawData` is the namespace ID of that share, `namespaceID`. -- The remaining [`SHARE_SIZE`](./consensus.md#constants)`-`[`NAMESPACE_ID_BYTES`](./consensus.md#constants) bytes are request data. In other words, the remaining bytes have no special meaning and are simply used to store data. +- If this is the first share of a message, the next 1-10 bytes contain a [varint](https://developers.google.com/protocol-buffers/docs/encoding) of the uint64 length of the message that follows. If this isn't the first share of a message, the next 1-10 bytes are used for request data. +- The remaining bytes are request data. In other words, the remaining bytes have no special meaning and are simply used to store data. For shares **with a namespace ID equal to [`PARITY_SHARE_NAMESPACE_ID`](./consensus.md#constants)** (i.e. parity shares): From 26f11cb235830345cce479cece6b366dd7770838 Mon Sep 17 00:00:00 2001 From: Rootul P Date: Mon, 31 Oct 2022 10:13:30 -0600 Subject: [PATCH 08/22] specs: universal share prefix (#856) Closes https://github.com/celestiaorg/celestia-app/issues/729 [Rendered](https://github.com/celestiaorg/celestia-app/blob/e6f00791df41a4257b677df52747e6453856770e/specs/src/specs/data_structures.md#share) Co-authored-by: John Adler --- specs/src/specs/consensus.md | 1 + specs/src/specs/data_structures.md | 46 +++++++++++++++---- .../figures/compact_continuation_share.dot | 27 +++++++++++ .../figures/compact_continuation_share.svg | 34 ++++++++++++++ .../src/specs/figures/compact_start_share.dot | 29 ++++++++++++ .../src/specs/figures/compact_start_share.svg | 37 +++++++++++++++ specs/src/specs/figures/share.dot | 25 ---------- specs/src/specs/figures/share.svg | 36 --------------- .../figures/sparse_continuation_share.dot | 21 +++++++++ .../figures/sparse_continuation_share.svg | 25 ++++++++++ .../src/specs/figures/sparse_start_share.dot | 21 +++++++++ .../src/specs/figures/sparse_start_share.svg | 25 ++++++++++ 12 files changed, 256 insertions(+), 71 deletions(-) create mode 100644 specs/src/specs/figures/compact_continuation_share.dot create mode 100644 specs/src/specs/figures/compact_continuation_share.svg create mode 100644 specs/src/specs/figures/compact_start_share.dot create mode 100644 specs/src/specs/figures/compact_start_share.svg delete mode 100644 specs/src/specs/figures/share.dot delete mode 100644 specs/src/specs/figures/share.svg create mode 100644 specs/src/specs/figures/sparse_continuation_share.dot create mode 100644 specs/src/specs/figures/sparse_continuation_share.svg create mode 100644 specs/src/specs/figures/sparse_start_share.dot create mode 100644 specs/src/specs/figures/sparse_start_share.svg diff --git a/specs/src/specs/consensus.md b/specs/src/specs/consensus.md index 71fdc7e5f2..ca6b00a8d9 100644 --- a/specs/src/specs/consensus.md +++ b/specs/src/specs/consensus.md @@ -55,6 +55,7 @@ | `MAX_VALIDATORS` | `uint16` | `64` | | Maximum number of active validators. | | `NAMESPACE_ID_BYTES` | `uint64` | `8` | `byte` | Size of namespace ID, in bytes. | | `NAMESPACE_ID_MAX_RESERVED` | `uint64` | `255` | | Value of maximum reserved namespace ID (inclusive). 1 byte worth of IDs. | +| `SHARE_INFO_BYTES` | `uint64` | `1` | `byte` | The number of bytes used for [share](data_structures.md#share) information | | `SHARE_RESERVED_BYTES` | `uint64` | `1` | `byte` | Bytes reserved at the beginning of each [share](data_structures.md#share). Must be sufficient to represent `SHARE_SIZE`. | | `SHARE_SIZE` | `uint64` | `256` | `byte` | Size of transaction and message [shares](data_structures.md#share), in bytes. | | `STATE_SUBTREE_RESERVED_BYTES` | `uint64` | `1` | `byte` | Number of bytes reserved to identify state subtrees. | diff --git a/specs/src/specs/data_structures.md b/specs/src/specs/data_structures.md index 0d0dc682b0..918d653bce 100644 --- a/specs/src/specs/data_structures.md +++ b/specs/src/specs/data_structures.md @@ -482,28 +482,54 @@ Finally, the `availableDataRoot` of the block [Header](#header) is computed as t A share is a fixed-size data chunk associated with a namespace ID, whose data will be erasure-coded and committed to in [Namespace Merkle trees](#namespace-merkle-tree). -A share's raw data `rawData` is interpreted differently depending on the namespace ID. +A sequence is a contiguous set of shares that contain semantically relevant data. A sequence should be parsed together because data may be split across share boundaries. One sequence exists per reserved namespace and per message. + +- The first [`NAMESPACE_ID_BYTES`](./consensus.md#constants) of a share's raw data `rawData` is the namespace ID of that share, `namespaceID`. +- The next [`SHARE_INFO_BYTES`](./consensus.md#constants) bytes are for share information with the following structure: + - The first 7 bits represent the share version in big endian form (initially, this will be `0000000` for version `0`); + - The last bit is a sequence start indicator, that is `1` if the share is at the start of a sequence or `0` if it is a continuation share. + +The remainder of a share's raw data `rawData` is interpreted differently depending on the namespace ID. + +#### Compact Share For shares **with a reserved namespace ID through [`NAMESPACE_ID_MAX_RESERVED`](./consensus.md#constants)**: -- The first [`NAMESPACE_ID_BYTES`](./consensus.md#constants) of a share's raw data `rawData` is the namespace ID of that share, `namespaceID`. -- The next [`SHARE_RESERVED_BYTES`](./consensus.md#constants) bytes (the `*` in the example layout figure below) is the starting byte of the length of the [canonically serialized](#serialization) first request that starts in the share, or `0` if there is none, as a one-byte big-endian unsigned integer (i.e. canonical serialization is not used). In this example, with a share size of `256` the first byte would be `80` (or `0x50` in hex). -- The remaining [`SHARE_SIZE`](./consensus.md#constants)`-`[`NAMESPACE_ID_BYTES`](./consensus.md#constants)`-`[`SHARE_RESERVED_BYTES`](./consensus.md#constants) bytes are request data. +> **Note** The first [`NAMESPACE_ID_BYTES`](./consensus.md#constants) of a share's raw data `rawData` is the namespace ID of that share, `namespaceID`. The next [`SHARE_INFO_BYTES`](./consensus.md#constants) bytes are for share information. + +- If this is the first share of a sequence, the next 1 to 10 bytes contain a [varint](https://developers.google.com/protocol-buffers/docs/encoding) of the `uint64` length of the sequence that follows, in bytes. +- The next [`SHARE_RESERVED_BYTES`](./consensus.md#constants) bytes is the starting byte of the length of the [canonically serialized](#serialization) first request that starts in the share, or `0` if there is none, as a one-byte big-endian unsigned integer (i.e. canonical serialization is not used). In the example below, with a share size of `256` the reserved byte would be `80` (or `0x50` in hex). +- The remaining [`SHARE_SIZE`](./consensus.md#constants)`-`[`NAMESPACE_ID_BYTES`](./consensus.md#constants)`-`[`SHARE_INFO_BYTES`](./consensus.md#constants) `-` 1 to 10 bytes (if this is the first share of a sequence) `-` [`SHARE_RESERVED_BYTES`](./consensus.md#constants) bytes are transactions, intermediate state roots, or evidence data depending on the namespace of ths share. Each transaction, intermediate state root, or evidence is prefixed with a [varint](https://developers.google.com/protocol-buffers/docs/encoding) of the length of that unit. +- If there is insufficient transaction, intermediate state root, or evidence data to fill the share, the remaining bytes are filled with `0`. + +First share in a sequence: +![fig: compact start share.](./figures/compact_start_share.svg) -![fig: Reserved share.](./figures/share.svg) +Continuation share in a sequence: +![fig: compact continuation share.](./figures/compact_continuation_share.svg) + +#### Sparse Share For shares **with a namespace ID above [`NAMESPACE_ID_MAX_RESERVED`](./consensus.md#constants) but below [`PARITY_SHARE_NAMESPACE_ID`](./consensus.md#constants)**: -- The first [`NAMESPACE_ID_BYTES`](./consensus.md#constants) of a share's raw data `rawData` is the namespace ID of that share, `namespaceID`. -- If this is the first share of a message, the next 1-10 bytes contain a [varint](https://developers.google.com/protocol-buffers/docs/encoding) of the uint64 length of the message that follows. If this isn't the first share of a message, the next 1-10 bytes are used for request data. -- The remaining bytes are request data. In other words, the remaining bytes have no special meaning and are simply used to store data. +> **Note** The first [`NAMESPACE_ID_BYTES`](./consensus.md#constants) of a share's raw data `rawData` is the namespace ID of that share, `namespaceID`. The next [`SHARE_INFO_BYTES`](./consensus.md#constants) bytes are for share information. + +- If this is the first share of a sequence, the next 1 to 10 bytes contain a [varint](https://developers.google.com/protocol-buffers/docs/encoding) of the `uint64` length of the sequence that follows. +- The remaining [`SHARE_SIZE`](./consensus.md#constants)`-`[`NAMESPACE_ID_BYTES`](./consensus.md#constants)`-`[`SHARE_INFO_BYTES`](./consensus.md#constants) `-` 1 to 10 bytes (if this is the first share of a sequence) bytes are message data. Message data are opaque bytes of data that are included in the block but do not impact the state. In other words, the remaining bytes have no special meaning and are simply used to store data. +- If there is insufficient message data to fill the share, the remaining bytes are filled with `0`. + +First share in a sequence: +![fig: sparse start share.](./figures/sparse_start_share.svg) + +Continuation share in a sequence: +![fig: sparse continuation share.](./figures/sparse_continuation_share.svg) + +#### Parity Share For shares **with a namespace ID equal to [`PARITY_SHARE_NAMESPACE_ID`](./consensus.md#constants)** (i.e. parity shares): - Bytes carry no special meaning. -For non-parity shares, if there is insufficient request data to fill the share, the remaining bytes are filled with `0`. - ### Arranging Available Data Into Shares The previous sections described how some original data, arranged into a `k * k` matrix, can be extended into a `2k * 2k` matrix and committed to with NMT roots. This section specifies how [available data](#available-data) (which includes [transactions](#transactiondata), [intermediate state roots](#intermediatestaterootdata), [evidence](#evidencedata), and [messages](#messagedata)) is arranged into the matrix in the first place. diff --git a/specs/src/specs/figures/compact_continuation_share.dot b/specs/src/specs/figures/compact_continuation_share.dot new file mode 100644 index 0000000000..9f33c14a16 --- /dev/null +++ b/specs/src/specs/figures/compact_continuation_share.dot @@ -0,0 +1,27 @@ +digraph G { + node [shape = record, penwidth = 0]; + + share [label=< + + + + + + + + + + + + + + + + + + + + +
089108082200256
namespace idinfo bytereserved byteend of tx2len(tx3)tx3zero-padding
+ >]; +} diff --git a/specs/src/specs/figures/compact_continuation_share.svg b/specs/src/specs/figures/compact_continuation_share.svg new file mode 100644 index 0000000000..3476739781 --- /dev/null +++ b/specs/src/specs/figures/compact_continuation_share.svg @@ -0,0 +1,34 @@ + + +G + + + +share + +0 +8 +9 +10 +80 +82 +200 +256 + +namespace id + +info byte + +reserved byte + +end of tx2 + +len(tx3) + +tx3 + +zero-padding + + + \ No newline at end of file diff --git a/specs/src/specs/figures/compact_start_share.dot b/specs/src/specs/figures/compact_start_share.dot new file mode 100644 index 0000000000..65f1585dc2 --- /dev/null +++ b/specs/src/specs/figures/compact_start_share.dot @@ -0,0 +1,29 @@ +digraph G { + node [shape = record, penwidth = 0]; + + share [label=< + + + + + + + + + + + + + + + + + + + + + + +
089121315200202256
namespace idinfo bytesequence lengthreserved bytelen(tx1)tx1len(tx2)tx2
+ >]; +} diff --git a/specs/src/specs/figures/compact_start_share.svg b/specs/src/specs/figures/compact_start_share.svg new file mode 100644 index 0000000000..e6a13e8160 --- /dev/null +++ b/specs/src/specs/figures/compact_start_share.svg @@ -0,0 +1,37 @@ + + +G + + + +share + +0 +8 +9 +12 +13 +15 +200 +202 +256 + +namespace id + +info byte + +sequence length + +reserved byte + +len(tx1) + +tx1 + +len(tx2) + +tx2 + + + \ No newline at end of file diff --git a/specs/src/specs/figures/share.dot b/specs/src/specs/figures/share.dot deleted file mode 100644 index f8ae292bc5..0000000000 --- a/specs/src/specs/figures/share.dot +++ /dev/null @@ -1,25 +0,0 @@ -digraph G { - node [shape = record, penwidth = 0]; - - share [label=< - - - - - - - - - - - - - - - - - - -
08980200256
NID*end of tx2len(tx3)tx3zero-padding
- >]; -} \ No newline at end of file diff --git a/specs/src/specs/figures/share.svg b/specs/src/specs/figures/share.svg deleted file mode 100644 index 02885ccc0e..0000000000 --- a/specs/src/specs/figures/share.svg +++ /dev/null @@ -1,36 +0,0 @@ - - - - - - -G - - - -share - -0 -8 -9 -80 -200 -256 - -NID - -* - -end of tx2 - -len(tx3) - -tx3 - -zero-padding - - - diff --git a/specs/src/specs/figures/sparse_continuation_share.dot b/specs/src/specs/figures/sparse_continuation_share.dot new file mode 100644 index 0000000000..73a4321bc2 --- /dev/null +++ b/specs/src/specs/figures/sparse_continuation_share.dot @@ -0,0 +1,21 @@ +digraph G { + node [shape = record, penwidth = 0]; + + share [label=< + + + + + + + + + + + + + + +
089200256
namespace idinfo byteend of msg1zero padding
+ >]; +} diff --git a/specs/src/specs/figures/sparse_continuation_share.svg b/specs/src/specs/figures/sparse_continuation_share.svg new file mode 100644 index 0000000000..0a1809d1b7 --- /dev/null +++ b/specs/src/specs/figures/sparse_continuation_share.svg @@ -0,0 +1,25 @@ + + +G + + + +share + +0 +8 +9 +200 +256 + +namespace id + +info byte + +end of msg1 + +zero padding + + + \ No newline at end of file diff --git a/specs/src/specs/figures/sparse_start_share.dot b/specs/src/specs/figures/sparse_start_share.dot new file mode 100644 index 0000000000..30e06b65f3 --- /dev/null +++ b/specs/src/specs/figures/sparse_start_share.dot @@ -0,0 +1,21 @@ +digraph G { + node [shape = record, penwidth = 0]; + + share [label=< + + + + + + + + + + + + + + +
08912256
namespace idinfo bytesequence lengthmsg1
+ >]; +} diff --git a/specs/src/specs/figures/sparse_start_share.svg b/specs/src/specs/figures/sparse_start_share.svg new file mode 100644 index 0000000000..5a87911685 --- /dev/null +++ b/specs/src/specs/figures/sparse_start_share.svg @@ -0,0 +1,25 @@ + + +G + + + +share + +0 +8 +9 +12 +256 + +namespace id + +info byte + +sequence length + +msg1 + + + \ No newline at end of file From 3206843b3548f04e035d7455a727ec527b22f9d8 Mon Sep 17 00:00:00 2001 From: Rootul P Date: Thu, 24 Nov 2022 13:40:05 -0700 Subject: [PATCH 09/22] specs: two reserved bytes (#939) Closes https://github.com/celestiaorg/celestia-app/issues/883 Co-authored-by: CHAMI Rachid --- specs/src/specs/consensus.md | 4 +- specs/src/specs/data_structures.md | 10 +++- .../figures/compact_continuation_share.dot | 6 +- .../figures/compact_continuation_share.svg | 50 ++++++++--------- .../src/specs/figures/compact_start_share.dot | 8 +-- .../src/specs/figures/compact_start_share.svg | 56 +++++++++---------- .../figures/sparse_continuation_share.dot | 2 +- .../figures/sparse_continuation_share.svg | 2 +- .../src/specs/figures/sparse_start_share.dot | 2 +- .../src/specs/figures/sparse_start_share.svg | 2 +- 10 files changed, 75 insertions(+), 67 deletions(-) diff --git a/specs/src/specs/consensus.md b/specs/src/specs/consensus.md index ca6b00a8d9..b10122f81e 100644 --- a/specs/src/specs/consensus.md +++ b/specs/src/specs/consensus.md @@ -56,8 +56,8 @@ | `NAMESPACE_ID_BYTES` | `uint64` | `8` | `byte` | Size of namespace ID, in bytes. | | `NAMESPACE_ID_MAX_RESERVED` | `uint64` | `255` | | Value of maximum reserved namespace ID (inclusive). 1 byte worth of IDs. | | `SHARE_INFO_BYTES` | `uint64` | `1` | `byte` | The number of bytes used for [share](data_structures.md#share) information | -| `SHARE_RESERVED_BYTES` | `uint64` | `1` | `byte` | Bytes reserved at the beginning of each [share](data_structures.md#share). Must be sufficient to represent `SHARE_SIZE`. | -| `SHARE_SIZE` | `uint64` | `256` | `byte` | Size of transaction and message [shares](data_structures.md#share), in bytes. | +| `SHARE_RESERVED_BYTES` | `uint64` | `2` | `byte` | Bytes reserved at the beginning of each [share](data_structures.md#share). Must be sufficient to represent `SHARE_SIZE`. | +| `SHARE_SIZE` | `uint64` | `512` | `byte` | Size of transaction and message [shares](data_structures.md#share), in bytes. | | `STATE_SUBTREE_RESERVED_BYTES` | `uint64` | `1` | `byte` | Number of bytes reserved to identify state subtrees. | | `UNBONDING_DURATION` | `uint32` | | `block` | Duration, in blocks, for unbonding a validator or delegation. | | `VERSION_APP` | `uint64` | `1` | | Version of the Celestia application. Breaking changes (hard forks) must update this parameter. | diff --git a/specs/src/specs/data_structures.md b/specs/src/specs/data_structures.md index 918d653bce..42aa2cdb8f 100644 --- a/specs/src/specs/data_structures.md +++ b/specs/src/specs/data_structures.md @@ -498,16 +498,22 @@ For shares **with a reserved namespace ID through [`NAMESPACE_ID_MAX_RESERVED`]( > **Note** The first [`NAMESPACE_ID_BYTES`](./consensus.md#constants) of a share's raw data `rawData` is the namespace ID of that share, `namespaceID`. The next [`SHARE_INFO_BYTES`](./consensus.md#constants) bytes are for share information. - If this is the first share of a sequence, the next 1 to 10 bytes contain a [varint](https://developers.google.com/protocol-buffers/docs/encoding) of the `uint64` length of the sequence that follows, in bytes. -- The next [`SHARE_RESERVED_BYTES`](./consensus.md#constants) bytes is the starting byte of the length of the [canonically serialized](#serialization) first request that starts in the share, or `0` if there is none, as a one-byte big-endian unsigned integer (i.e. canonical serialization is not used). In the example below, with a share size of `256` the reserved byte would be `80` (or `0x50` in hex). +- The next [`SHARE_RESERVED_BYTES`](./consensus.md#constants) bytes are the starting byte of the length of the [canonically serialized](#serialization) first request that starts in the share, or `0` if there is none, as an unsigned [varint](https://developers.google.com/protocol-buffers/docs/encoding). - The remaining [`SHARE_SIZE`](./consensus.md#constants)`-`[`NAMESPACE_ID_BYTES`](./consensus.md#constants)`-`[`SHARE_INFO_BYTES`](./consensus.md#constants) `-` 1 to 10 bytes (if this is the first share of a sequence) `-` [`SHARE_RESERVED_BYTES`](./consensus.md#constants) bytes are transactions, intermediate state roots, or evidence data depending on the namespace of ths share. Each transaction, intermediate state root, or evidence is prefixed with a [varint](https://developers.google.com/protocol-buffers/docs/encoding) of the length of that unit. - If there is insufficient transaction, intermediate state root, or evidence data to fill the share, the remaining bytes are filled with `0`. First share in a sequence: + ![fig: compact start share.](./figures/compact_start_share.svg) +where reserved bytes would be `14` encoded as an unsigned varint (`[0b00001110, 0b00000000]`). + Continuation share in a sequence: + ![fig: compact continuation share.](./figures/compact_continuation_share.svg) +where reserved bytes would be `80` encoded as an unsigned varint (`[0b01010000, 0b00000000]`). + #### Sparse Share For shares **with a namespace ID above [`NAMESPACE_ID_MAX_RESERVED`](./consensus.md#constants) but below [`PARITY_SHARE_NAMESPACE_ID`](./consensus.md#constants)**: @@ -519,9 +525,11 @@ For shares **with a namespace ID above [`NAMESPACE_ID_MAX_RESERVED`](./consensus - If there is insufficient message data to fill the share, the remaining bytes are filled with `0`. First share in a sequence: + ![fig: sparse start share.](./figures/sparse_start_share.svg) Continuation share in a sequence: + ![fig: sparse continuation share.](./figures/sparse_continuation_share.svg) #### Parity Share diff --git a/specs/src/specs/figures/compact_continuation_share.dot b/specs/src/specs/figures/compact_continuation_share.dot index 9f33c14a16..090fded692 100644 --- a/specs/src/specs/figures/compact_continuation_share.dot +++ b/specs/src/specs/figures/compact_continuation_share.dot @@ -7,16 +7,16 @@ digraph G { 0 8 9 - 10 + 11 80 82 200 - 256 + 512 namespace id info byte - reserved byte + reserved bytes end of tx2 len(tx3) tx3 diff --git a/specs/src/specs/figures/compact_continuation_share.svg b/specs/src/specs/figures/compact_continuation_share.svg index 3476739781..36909cc8f1 100644 --- a/specs/src/specs/figures/compact_continuation_share.svg +++ b/specs/src/specs/figures/compact_continuation_share.svg @@ -1,34 +1,34 @@ + --> G - + share - -0 -8 -9 -10 -80 -82 -200 -256 - -namespace id - -info byte - -reserved byte - -end of tx2 - -len(tx3) - -tx3 - -zero-padding + +0 +8 +9 +11 +80 +82 +200 +512 + +namespace id + +info byte + +reserved bytes + +end of tx2 + +len(tx3) + +tx3 + +zero-padding \ No newline at end of file diff --git a/specs/src/specs/figures/compact_start_share.dot b/specs/src/specs/figures/compact_start_share.dot index 65f1585dc2..dfe6120c72 100644 --- a/specs/src/specs/figures/compact_start_share.dot +++ b/specs/src/specs/figures/compact_start_share.dot @@ -8,17 +8,17 @@ digraph G { 8 9 12 - 13 - 15 + 14 + 16 200 202 - 256 + 512
namespace id info byte sequence length - reserved byte + reserved bytes len(tx1) tx1 len(tx2) diff --git a/specs/src/specs/figures/compact_start_share.svg b/specs/src/specs/figures/compact_start_share.svg index e6a13e8160..fcdadeb4a0 100644 --- a/specs/src/specs/figures/compact_start_share.svg +++ b/specs/src/specs/figures/compact_start_share.svg @@ -1,37 +1,37 @@ + --> G - + share - -0 -8 -9 -12 -13 -15 -200 -202 -256 - -namespace id - -info byte - -sequence length - -reserved byte - -len(tx1) - -tx1 - -len(tx2) - -tx2 + +0 +8 +9 +12 +14 +16 +200 +202 +512 + +namespace id + +info byte + +sequence length + +reserved bytes + +len(tx1) + +tx1 + +len(tx2) + +tx2 \ No newline at end of file diff --git a/specs/src/specs/figures/sparse_continuation_share.dot b/specs/src/specs/figures/sparse_continuation_share.dot index 73a4321bc2..d8470f0bf6 100644 --- a/specs/src/specs/figures/sparse_continuation_share.dot +++ b/specs/src/specs/figures/sparse_continuation_share.dot @@ -8,7 +8,7 @@ digraph G { 8 9 200 - 256 + 512
namespace id diff --git a/specs/src/specs/figures/sparse_continuation_share.svg b/specs/src/specs/figures/sparse_continuation_share.svg index 0a1809d1b7..af746cd1d9 100644 --- a/specs/src/specs/figures/sparse_continuation_share.svg +++ b/specs/src/specs/figures/sparse_continuation_share.svg @@ -11,7 +11,7 @@ 8 9 200 -256 +512 namespace id diff --git a/specs/src/specs/figures/sparse_start_share.dot b/specs/src/specs/figures/sparse_start_share.dot index 30e06b65f3..da8915fde4 100644 --- a/specs/src/specs/figures/sparse_start_share.dot +++ b/specs/src/specs/figures/sparse_start_share.dot @@ -8,7 +8,7 @@ digraph G { 8 9 12 - 256 + 512
namespace id diff --git a/specs/src/specs/figures/sparse_start_share.svg b/specs/src/specs/figures/sparse_start_share.svg index 5a87911685..3b4bc9be76 100644 --- a/specs/src/specs/figures/sparse_start_share.svg +++ b/specs/src/specs/figures/sparse_start_share.svg @@ -11,7 +11,7 @@ 8 9 12 -256 +512 namespace id From b610a0b3052a2451871e16b097b71816681b6b8a Mon Sep 17 00:00:00 2001 From: Rootul P Date: Fri, 6 Jan 2023 07:24:39 -0700 Subject: [PATCH 10/22] chore: uint32 for message size (#1182) Part of https://github.com/celestiaorg/celestia-app/issues/1153 Should only be merged if https://github.com/celestiaorg/celestia-app/pull/1181 is merged --- specs/src/specs/data_structures.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/src/specs/data_structures.md b/specs/src/specs/data_structures.md index 42aa2cdb8f..941335a6b4 100644 --- a/specs/src/specs/data_structures.md +++ b/specs/src/specs/data_structures.md @@ -669,7 +669,7 @@ Transfers `amount` coins to `to`. | `fee` | [TransactionFee](#transactionfee) | | | `nonce` | [Nonce](#type-aliases) | | | `messageNamespaceID` | [`NamespaceID`](#type-aliases) | Namespace ID of message this transaction pays a fee for. | -| `messageSize` | `uint64` | Size of message this transaction pays a fee for, in `byte`s. | +| `messageSize` | `uint32` | Size of message this transaction pays a fee for, in `byte`s. | | `messageShareCommitment` | [HashDigest](#hashdigest) | Commitment to message shares (details below). | Pays for the inclusion of a [message](#message) in the same block. From 351e2d3627ffe3e7ab8694096b8d0b174a88e67e Mon Sep 17 00:00:00 2001 From: Rootul P Date: Wed, 11 Jan 2023 09:02:12 -0700 Subject: [PATCH 11/22] specs: fixed sequence length (#1124) Closes https://github.com/celestiaorg/celestia-app/issues/1120 Co-authored-by: John Adler --- specs/src/specs/consensus.md | 3 ++- specs/src/specs/data_structures.md | 12 ++++++------ .../src/specs/figures/compact_continuation_share.dot | 4 ++-- .../src/specs/figures/compact_continuation_share.svg | 4 ++-- specs/src/specs/figures/compact_start_share.dot | 8 ++++---- specs/src/specs/figures/compact_start_share.svg | 6 +++--- specs/src/specs/figures/sparse_start_share.dot | 2 +- specs/src/specs/figures/sparse_start_share.svg | 2 +- 8 files changed, 21 insertions(+), 20 deletions(-) diff --git a/specs/src/specs/consensus.md b/specs/src/specs/consensus.md index b10122f81e..c870ddadf0 100644 --- a/specs/src/specs/consensus.md +++ b/specs/src/specs/consensus.md @@ -55,8 +55,9 @@ | `MAX_VALIDATORS` | `uint16` | `64` | | Maximum number of active validators. | | `NAMESPACE_ID_BYTES` | `uint64` | `8` | `byte` | Size of namespace ID, in bytes. | | `NAMESPACE_ID_MAX_RESERVED` | `uint64` | `255` | | Value of maximum reserved namespace ID (inclusive). 1 byte worth of IDs. | +| `SEQUENCE_BYTES` | `uint64` | `4` | `byte` | The number of bytes used to store the sequence length in the first share of a sequence | | `SHARE_INFO_BYTES` | `uint64` | `1` | `byte` | The number of bytes used for [share](data_structures.md#share) information | -| `SHARE_RESERVED_BYTES` | `uint64` | `2` | `byte` | Bytes reserved at the beginning of each [share](data_structures.md#share). Must be sufficient to represent `SHARE_SIZE`. | +| `SHARE_RESERVED_BYTES` | `uint64` | `4` | `byte` | The number of bytes used to store the location of the first unit in a compact share. Must be able to represent any integer up to and including `SHARE_SIZE - 1`. | | `SHARE_SIZE` | `uint64` | `512` | `byte` | Size of transaction and message [shares](data_structures.md#share), in bytes. | | `STATE_SUBTREE_RESERVED_BYTES` | `uint64` | `1` | `byte` | Number of bytes reserved to identify state subtrees. | | `UNBONDING_DURATION` | `uint32` | | `block` | Duration, in blocks, for unbonding a validator or delegation. | diff --git a/specs/src/specs/data_structures.md b/specs/src/specs/data_structures.md index 941335a6b4..3bfd1ab8cd 100644 --- a/specs/src/specs/data_structures.md +++ b/specs/src/specs/data_structures.md @@ -497,22 +497,22 @@ For shares **with a reserved namespace ID through [`NAMESPACE_ID_MAX_RESERVED`]( > **Note** The first [`NAMESPACE_ID_BYTES`](./consensus.md#constants) of a share's raw data `rawData` is the namespace ID of that share, `namespaceID`. The next [`SHARE_INFO_BYTES`](./consensus.md#constants) bytes are for share information. -- If this is the first share of a sequence, the next 1 to 10 bytes contain a [varint](https://developers.google.com/protocol-buffers/docs/encoding) of the `uint64` length of the sequence that follows, in bytes. +- If this is the first share of a sequence, the next [`SEQUENCE_BYTES`](./consensus.md#constants) contain a big endian `uint32` that represents the length of the sequence that follows in bytes. - The next [`SHARE_RESERVED_BYTES`](./consensus.md#constants) bytes are the starting byte of the length of the [canonically serialized](#serialization) first request that starts in the share, or `0` if there is none, as an unsigned [varint](https://developers.google.com/protocol-buffers/docs/encoding). -- The remaining [`SHARE_SIZE`](./consensus.md#constants)`-`[`NAMESPACE_ID_BYTES`](./consensus.md#constants)`-`[`SHARE_INFO_BYTES`](./consensus.md#constants) `-` 1 to 10 bytes (if this is the first share of a sequence) `-` [`SHARE_RESERVED_BYTES`](./consensus.md#constants) bytes are transactions, intermediate state roots, or evidence data depending on the namespace of ths share. Each transaction, intermediate state root, or evidence is prefixed with a [varint](https://developers.google.com/protocol-buffers/docs/encoding) of the length of that unit. +- The remaining [`SHARE_SIZE`](./consensus.md#constants)`-`[`NAMESPACE_ID_BYTES`](./consensus.md#constants)`-`[`SHARE_INFO_BYTES`](./consensus.md#constants) `-` [`SEQUENCE_BYTES`](.consensus.md#constants) bytes (only if this is the first share of a sequence) `-` [`SHARE_RESERVED_BYTES`](./consensus.md#constants) bytes are transactions, intermediate state roots, or evidence data depending on the namespace of ths share. Each transaction, intermediate state root, or evidence is prefixed with a [varint](https://developers.google.com/protocol-buffers/docs/encoding) of the length of that unit. - If there is insufficient transaction, intermediate state root, or evidence data to fill the share, the remaining bytes are filled with `0`. First share in a sequence: ![fig: compact start share.](./figures/compact_start_share.svg) -where reserved bytes would be `14` encoded as an unsigned varint (`[0b00001110, 0b00000000]`). +where reserved bytes would be `17` as a binary big endian `uint32` (`[0b00000000, 0b00000000, 0b00000000, 0b00010001]`). Continuation share in a sequence: ![fig: compact continuation share.](./figures/compact_continuation_share.svg) -where reserved bytes would be `80` encoded as an unsigned varint (`[0b01010000, 0b00000000]`). +where reserved bytes would be `80` as a binary big endian `uint32` (`[0b00000000, 0b00000000, 0b00000000, 0b01010000]`). #### Sparse Share @@ -520,8 +520,8 @@ For shares **with a namespace ID above [`NAMESPACE_ID_MAX_RESERVED`](./consensus > **Note** The first [`NAMESPACE_ID_BYTES`](./consensus.md#constants) of a share's raw data `rawData` is the namespace ID of that share, `namespaceID`. The next [`SHARE_INFO_BYTES`](./consensus.md#constants) bytes are for share information. -- If this is the first share of a sequence, the next 1 to 10 bytes contain a [varint](https://developers.google.com/protocol-buffers/docs/encoding) of the `uint64` length of the sequence that follows. -- The remaining [`SHARE_SIZE`](./consensus.md#constants)`-`[`NAMESPACE_ID_BYTES`](./consensus.md#constants)`-`[`SHARE_INFO_BYTES`](./consensus.md#constants) `-` 1 to 10 bytes (if this is the first share of a sequence) bytes are message data. Message data are opaque bytes of data that are included in the block but do not impact the state. In other words, the remaining bytes have no special meaning and are simply used to store data. +- If this is the first share of a sequence, the next [`SEQUENCE_BYTES`](./consensus.md#constants) contain a big endian `uint32` that represents the length of the sequence that follows in bytes. +- The remaining [`SHARE_SIZE`](./consensus.md#constants)`-`[`NAMESPACE_ID_BYTES`](./consensus.md#constants)`-`[`SHARE_INFO_BYTES`](./consensus.md#constants) `-` [`SEQUENCE_BYTES`](.consensus.md#constants) bytes (only if this is the first share of a sequence) bytes are message data. Message data are opaque bytes of data that are included in the block but do not impact the state. In other words, the remaining bytes have no special meaning and are simply used to store data. - If there is insufficient message data to fill the share, the remaining bytes are filled with `0`. First share in a sequence: diff --git a/specs/src/specs/figures/compact_continuation_share.dot b/specs/src/specs/figures/compact_continuation_share.dot index 090fded692..1d9699f462 100644 --- a/specs/src/specs/figures/compact_continuation_share.dot +++ b/specs/src/specs/figures/compact_continuation_share.dot @@ -7,7 +7,7 @@ digraph G { 0 8 9 - 11 + 13 80 82 200 @@ -16,7 +16,7 @@ digraph G {
namespace id info byte - reserved bytes + reserved bytes end of tx2 len(tx3) tx3 diff --git a/specs/src/specs/figures/compact_continuation_share.svg b/specs/src/specs/figures/compact_continuation_share.svg index 36909cc8f1..c2d85e3ff1 100644 --- a/specs/src/specs/figures/compact_continuation_share.svg +++ b/specs/src/specs/figures/compact_continuation_share.svg @@ -10,7 +10,7 @@ 0 8 9 -11 +13 80 82 200 @@ -31,4 +31,4 @@ zero-padding -
 \ No newline at end of file + diff --git a/specs/src/specs/figures/compact_start_share.dot b/specs/src/specs/figures/compact_start_share.dot index dfe6120c72..f35182200f 100644 --- a/specs/src/specs/figures/compact_start_share.dot +++ b/specs/src/specs/figures/compact_start_share.dot @@ -7,9 +7,9 @@ digraph G { 0 8 9 - 12 - 14 - 16 + 13 + 17 + 18 200 202 512 @@ -18,7 +18,7 @@ digraph G { namespace id info byte sequence length - reserved bytes + reserved bytes len(tx1) tx1 len(tx2) diff --git a/specs/src/specs/figures/compact_start_share.svg b/specs/src/specs/figures/compact_start_share.svg index fcdadeb4a0..1a0f14a950 100644 --- a/specs/src/specs/figures/compact_start_share.svg +++ b/specs/src/specs/figures/compact_start_share.svg @@ -10,9 +10,9 @@ 0 8 9 -12 -14 -16 +13 +17 +18 200 202 512 diff --git a/specs/src/specs/figures/sparse_start_share.dot b/specs/src/specs/figures/sparse_start_share.dot index da8915fde4..28253354a7 100644 --- a/specs/src/specs/figures/sparse_start_share.dot +++ b/specs/src/specs/figures/sparse_start_share.dot @@ -7,7 +7,7 @@ digraph G { 0 8 9 - 12 + 13 512 diff --git a/specs/src/specs/figures/sparse_start_share.svg b/specs/src/specs/figures/sparse_start_share.svg index 3b4bc9be76..34a878f5f0 100644 --- a/specs/src/specs/figures/sparse_start_share.svg +++ b/specs/src/specs/figures/sparse_start_share.svg @@ -10,7 +10,7 @@ 0 8 9 -12 +13 512 namespace id From 41c0859a44b0dfcf11d4e691f35978a2d4bfc1f3 Mon Sep 17 00:00:00 2001 From: Rootul P Date: Thu, 12 Jan 2023 08:28:08 -0700 Subject: [PATCH 12/22] specs: non-interactive default rules for reduced padding (#1156) Closes https://github.com/celestiaorg/celestia-app/issues/1151 Co-authored-by: John Adler --- specs/src/rationale/message_block_layout.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/specs/src/rationale/message_block_layout.md b/specs/src/rationale/message_block_layout.md index e8b7757901..1af44b31e2 100644 --- a/specs/src/rationale/message_block_layout.md +++ b/specs/src/rationale/message_block_layout.md @@ -35,14 +35,13 @@ This, however, requires the block producer to interact with the transaction send ### Non-Interactive Default Rules -As a non-consensus-critical default, we can impose some additional rules on message placement to make the possible starting locations of messages sufficiently predictable and constrained such that users can deterministically compute subtree roots without interaction: +As a non-consensus-critical default, we can impose one additional rule on message placement to make the possible starting locations of messages sufficiently predictable and constrained such that users can deterministically compute subtree roots without interaction: -1. Messages that span multiple rows must begin at the start of a row (this can occur if a message is longer than `k` shares _or_ if the block producer decides to start a message partway through a row and it cannot fit). -1. Messages begin at a location aligned with the largest power of 2 that is not larger than the message length or `k`. +> Messages start at an index that is a multiple of the message minimum square size. The message minimum square size is the smallest square that can contain the message in isolation (i.e. a square with only this message and no other transactions or messages). -With the above constraints, we can compute subtree roots deterministically easily: simply slice off either the largest power of 2 that isn't larger than the remaining message length, or `k`, whichever is smaller. This is done recursively. As an example, with `k = 4` and message length of `11`, the message would be sliced with lengths `4, 4, 2, 1`. The resulting slices are the leaves of subtrees whose roots can be computed. Due to the rules above, the subtrees are guaranteed to be aligned to powers of 2, and thus the subtree roots will be present as internal nodes in the NMT of _some_ row(s). +With the above constraint, we can compute subtree roots deterministically. In order to compute the subtree roots, split the message into chunks that are of maximum size: message minimum square size. As an example, a message of length `11` has a minimum square size of `4` because `11` is not greater than `4 * 4 = 16` total shares. Split the message into chunks of length `4, 4, 2, 1`. The resulting slices are the leaves of subtrees whose roots can be computed. These subtree roots will be present as internal nodes in the NMT of _some_ row(s). -This is similar to [Merkle Mountain Ranges](https://www.usenix.org/legacy/event/sec09/tech/full_papers/crosby.pdf), though with the largest subtree bounded by `k` rather than being unbounded. +This is similar to [Merkle Mountain Ranges](https://www.usenix.org/legacy/event/sec09/tech/full_papers/crosby.pdf), though with the largest subtree bounded by the message minimum square size rather than being unbounded. The last piece of the puzzle is determining _which_ row the message is placed at (or, more specifically, the starting location). This is needed to keep the block producer accountable. To this end, the block producer simply augments each fee-paying transaction with some metadata: the starting location of the message the transaction pays for. From 3c66c2d3e79a652b9facfad9b6202de7d0738ce8 Mon Sep 17 00:00:00 2001 From: Rootul P Date: Tue, 17 Jan 2023 09:01:21 -0700 Subject: [PATCH 13/22] docs: spec namespaced padding share (#1248) Closes https://github.com/celestiaorg/celestia-app/issues/1247 --- specs/src/specs/data_structures.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/specs/src/specs/data_structures.md b/specs/src/specs/data_structures.md index 3bfd1ab8cd..df1580a27c 100644 --- a/specs/src/specs/data_structures.md +++ b/specs/src/specs/data_structures.md @@ -29,6 +29,10 @@ - [Reed-Solomon Erasure Coding](#reed-solomon-erasure-coding) - [2D Reed-Solomon Encoding Scheme](#2d-reed-solomon-encoding-scheme) - [Share](#share) + - [Compact Share](#compact-share) + - [Sparse Share](#sparse-share) + - [Parity Share](#parity-share) + - [Namespaced Padding Share](#namespaced-padding-share) - [Arranging Available Data Into Shares](#arranging-available-data-into-shares) - [Available Data](#available-data) - [TransactionData](#transactiondata) @@ -538,6 +542,12 @@ For shares **with a namespace ID equal to [`PARITY_SHARE_NAMESPACE_ID`](./consen - Bytes carry no special meaning. +#### Namespaced Padding Share + +A namespaced padding share acts as padding between blobs so that the subsequent blob may begin at an index that conforms to the [non-interactive default rules](../rationale/message_block_layout.md#non-interactive-default-rules). A namespaced padding share contains the namespace ID of the blob that precedes it in the data square so that the data square can retain the property that all shares are ordered by namespace. + +The first [`NAMESPACE_ID_BYTES`](./consensus.md#constants) of a share's raw data `rawData` is the namespace ID of the blob that precedes this padding share. The next [`SHARE_INFO_BYTES`](./consensus.md#constants) bytes are for share information. The sequence start indicator is always `0`. The version bits are filled with the share version. The remaining [`SHARE_SIZE`](./consensus.md#constants)`-`[`NAMESPACE_ID_BYTES`](./consensus.md#constants)`-`[`SHARE_INFO_BYTES`](./consensus.md#constants) bytes are filled with `0`. + ### Arranging Available Data Into Shares The previous sections described how some original data, arranged into a `k * k` matrix, can be extended into a `2k * 2k` matrix and committed to with NMT roots. This section specifies how [available data](#available-data) (which includes [transactions](#transactiondata), [intermediate state roots](#intermediatestaterootdata), [evidence](#evidencedata), and [messages](#messagedata)) is arranged into the matrix in the first place. From 4f83347697f063ee33a37e8473537d6557632aa0 Mon Sep 17 00:00:00 2001 From: Rootul P Date: Wed, 18 Jan 2023 08:47:49 -0700 Subject: [PATCH 14/22] docs: spec tail padding share (#1244) Closes https://github.com/celestiaorg/celestia-app/issues/1175 See the implementation [here](https://github.com/celestiaorg/celestia-app/blob/d7ac0b84765db3f649cd82326e3ead066d164637/pkg/shares/tail_padding.go#L9-L21). ## Screenshot Screenshot 2023-01-17 at 11 04 14 AM --- specs/src/specs/data_structures.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/specs/src/specs/data_structures.md b/specs/src/specs/data_structures.md index df1580a27c..c9cde46f8d 100644 --- a/specs/src/specs/data_structures.md +++ b/specs/src/specs/data_structures.md @@ -33,6 +33,7 @@ - [Sparse Share](#sparse-share) - [Parity Share](#parity-share) - [Namespaced Padding Share](#namespaced-padding-share) + - [Tail Padding Share](#tail-padding-share) - [Arranging Available Data Into Shares](#arranging-available-data-into-shares) - [Available Data](#available-data) - [TransactionData](#transactiondata) @@ -548,6 +549,14 @@ A namespaced padding share acts as padding between blobs so that the subsequent The first [`NAMESPACE_ID_BYTES`](./consensus.md#constants) of a share's raw data `rawData` is the namespace ID of the blob that precedes this padding share. The next [`SHARE_INFO_BYTES`](./consensus.md#constants) bytes are for share information. The sequence start indicator is always `0`. The version bits are filled with the share version. The remaining [`SHARE_SIZE`](./consensus.md#constants)`-`[`NAMESPACE_ID_BYTES`](./consensus.md#constants)`-`[`SHARE_INFO_BYTES`](./consensus.md#constants) bytes are filled with `0`. +#### Tail Padding Share + +Tail padding shares are placed after the last blob in the data square so that the number of shares in the data square is a perfect square. Clients can safely ignore the contents of these shares because they don't contain any significant data. + +For shares **with a namespace ID equal to [`TAIL_PADDING_NAMESPACE_ID`](./consensus.md#constants)** (i.e. tail padding shares): + +The first [`NAMESPACE_ID_BYTES`](./consensus.md#constants) of a share's raw data `rawData` is the namespace ID of that share, `namespaceID`. The next [`SHARE_INFO_BYTES`](./consensus.md#constants) bytes are for share information. The sequence start indicator is always `0`. The version bits are filled with the share version. The remaining [`SHARE_SIZE`](./consensus.md#constants)`-`[`NAMESPACE_ID_BYTES`](./consensus.md#constants)`-`[`SHARE_INFO_BYTES`](./consensus.md#constants) bytes are filled with `0`. + ### Arranging Available Data Into Shares The previous sections described how some original data, arranged into a `k * k` matrix, can be extended into a `2k * 2k` matrix and committed to with NMT roots. This section specifies how [available data](#available-data) (which includes [transactions](#transactiondata), [intermediate state roots](#intermediatestaterootdata), [evidence](#evidencedata), and [messages](#messagedata)) is arranged into the matrix in the first place. From 0dc7d17636efa535efca42af58229ee0c3c21261 Mon Sep 17 00:00:00 2001 From: Rootul P Date: Wed, 8 Feb 2023 13:28:32 -0700 Subject: [PATCH 15/22] docs: revise specs for padding shares (#1359) Context: https://github.com/celestiaorg/celestia-app/pull/1353 Closes https://github.com/celestiaorg/celestia-app/issues/1344 ## Screenshot Screenshot 2023-02-08 at 11 55 07 AM --- specs/src/specs/consensus.md | 2 +- specs/src/specs/data_structures.md | 18 +++++++++++++----- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/specs/src/specs/consensus.md b/specs/src/specs/consensus.md index c870ddadf0..64ce27eb9d 100644 --- a/specs/src/specs/consensus.md +++ b/specs/src/specs/consensus.md @@ -71,7 +71,7 @@ | `TRANSACTION_NAMESPACE_ID` | `NamespaceID` | `0x0000000000000001` | Transactions: requests that modify the state. | | `INTERMEDIATE_STATE_ROOT_NAMESPACE_ID` | `NamespaceID` | `0x0000000000000002` | Intermediate state roots, committed after every transaction. | | `EVIDENCE_NAMESPACE_ID` | `NamespaceID` | `0x0000000000000003` | Evidence: fraud proofs or other proof of slashable action. | -| `TAIL_TRANSACTION_PADDING_NAMESPACE_ID` | `NamespaceID` | `0x00000000000000FF` | Tail padding for transactions: padding after all transactions but before messages. | +| `RESERVED_PADDING_NAMESPACE_ID` | `NamespaceID` | `0x00000000000000FF` | Padding after all reserved namespaces but before blobs. | | `TAIL_PADDING_NAMESPACE_ID` | `NamespaceID` | `0xFFFFFFFFFFFFFFFE` | Tail padding for messages: padding after all messages to fill up the original data square. | | `PARITY_SHARE_NAMESPACE_ID` | `NamespaceID` | `0xFFFFFFFFFFFFFFFF` | Parity shares: extended shares in the available data matrix. | diff --git a/specs/src/specs/data_structures.md b/specs/src/specs/data_structures.md index c9cde46f8d..8800c3bcbd 100644 --- a/specs/src/specs/data_structures.md +++ b/specs/src/specs/data_structures.md @@ -32,7 +32,7 @@ - [Compact Share](#compact-share) - [Sparse Share](#sparse-share) - [Parity Share](#parity-share) - - [Namespaced Padding Share](#namespaced-padding-share) + - [Namespace Padding Share](#namespace-padding-share) - [Tail Padding Share](#tail-padding-share) - [Arranging Available Data Into Shares](#arranging-available-data-into-shares) - [Available Data](#available-data) @@ -543,11 +543,19 @@ For shares **with a namespace ID equal to [`PARITY_SHARE_NAMESPACE_ID`](./consen - Bytes carry no special meaning. -#### Namespaced Padding Share +#### Namespace Padding Share -A namespaced padding share acts as padding between blobs so that the subsequent blob may begin at an index that conforms to the [non-interactive default rules](../rationale/message_block_layout.md#non-interactive-default-rules). A namespaced padding share contains the namespace ID of the blob that precedes it in the data square so that the data square can retain the property that all shares are ordered by namespace. +A namespace padding share acts as padding between blobs so that the subsequent blob may begin at an index that conforms to the [non-interactive default rules](../rationale/message_block_layout.md#non-interactive-default-rules). A namespace padding share contains the namespace ID of the blob that precedes it in the data square so that the data square can retain the property that all shares are ordered by namespace. -The first [`NAMESPACE_ID_BYTES`](./consensus.md#constants) of a share's raw data `rawData` is the namespace ID of the blob that precedes this padding share. The next [`SHARE_INFO_BYTES`](./consensus.md#constants) bytes are for share information. The sequence start indicator is always `0`. The version bits are filled with the share version. The remaining [`SHARE_SIZE`](./consensus.md#constants)`-`[`NAMESPACE_ID_BYTES`](./consensus.md#constants)`-`[`SHARE_INFO_BYTES`](./consensus.md#constants) bytes are filled with `0`. +The first [`NAMESPACE_ID_BYTES`](./consensus.md#constants) of a share's raw data `rawData` is the namespace ID of the blob that precedes this padding share. The next [`SHARE_INFO_BYTES`](./consensus.md#constants) bytes are for share information. The sequence start indicator is always `1`. The version bits are filled with the share version. The sequence length is zeroed out. The remaining [`SHARE_SIZE`](./consensus.md#constants)`-`[`NAMESPACE_ID_BYTES`](./consensus.md#constants)`-`[`SHARE_INFO_BYTES`](./consensus.md#constants) `-` [`SEQUENCE_BYTES`](./consensus.md#constants) bytes are filled with `0`. + +#### Reserved Padding Share + +Reserved padding shares are placed after the last reserved namespace share in the data square so that the first blob can start at an index that conforms to non-interactive default rules. Clients can safely ignore the contents of these shares because they don't contain any significant data. + +For shares **with a namespace ID equal to [`RESERVED_PADDING_NAMESPACE_ID`](./consensus.md#constants)** (i.e. reserved padding shares): + +The first [`NAMESPACE_ID_BYTES`](./consensus.md#constants) of a share's raw data `rawData` is the namespace ID of that share, `namespaceID`. The next [`SHARE_INFO_BYTES`](./consensus.md#constants) bytes are for share information. The sequence start indicator is always `1`. The version bits are filled with the share version. The sequence length is zeroed out. The remaining [`SHARE_SIZE`](./consensus.md#constants)`-`[`NAMESPACE_ID_BYTES`](./consensus.md#constants)`-`[`SHARE_INFO_BYTES`](./consensus.md#constants) `-` [`SEQUENCE_BYTES`](./consensus.md#constants) bytes are filled with `0`. #### Tail Padding Share @@ -555,7 +563,7 @@ Tail padding shares are placed after the last blob in the data square so that th For shares **with a namespace ID equal to [`TAIL_PADDING_NAMESPACE_ID`](./consensus.md#constants)** (i.e. tail padding shares): -The first [`NAMESPACE_ID_BYTES`](./consensus.md#constants) of a share's raw data `rawData` is the namespace ID of that share, `namespaceID`. The next [`SHARE_INFO_BYTES`](./consensus.md#constants) bytes are for share information. The sequence start indicator is always `0`. The version bits are filled with the share version. The remaining [`SHARE_SIZE`](./consensus.md#constants)`-`[`NAMESPACE_ID_BYTES`](./consensus.md#constants)`-`[`SHARE_INFO_BYTES`](./consensus.md#constants) bytes are filled with `0`. +The first [`NAMESPACE_ID_BYTES`](./consensus.md#constants) of a share's raw data `rawData` is the namespace ID of that share, `namespaceID`. The next [`SHARE_INFO_BYTES`](./consensus.md#constants) bytes are for share information. The sequence start indicator is always `1`. The version bits are filled with the share version. The sequence length is zeroed out. The remaining [`SHARE_SIZE`](./consensus.md#constants)`-`[`NAMESPACE_ID_BYTES`](./consensus.md#constants)`-`[`SHARE_INFO_BYTES`](./consensus.md#constants) `-` [`SEQUENCE_BYTES`](./consensus.md#constants) bytes are filled with `0`. ### Arranging Available Data Into Shares From 329dc50c51ee4865439ef3ef2d466c0ab61bcc28 Mon Sep 17 00:00:00 2001 From: Sanaz Taheri <35961250+staheri14@users.noreply.github.com> Date: Thu, 23 Feb 2023 09:31:35 -0800 Subject: [PATCH 16/22] chore: specifies minimum square size in non-interactive default rules (#1420) This aims to clarify that the minimum square size in non-interactive default rules should be a power of 2. For example, for a message of length 23, the minimum square size is `8*8` (but not `5*5`). (based on a discussion in Slack) - [x] NA ~~New and updated code has appropriate documentation~~ - [x] NA ~~New and updated code has new and/or updated testing~~ - [x] Required CI checks are passing - [x] NA ~~Visual proof for any user facing features like CLI or documentation updates~~ - [x] NA ~~Linked issues closed with keywords~~ --- specs/src/rationale/message_block_layout.md | 1 + 1 file changed, 1 insertion(+) diff --git a/specs/src/rationale/message_block_layout.md b/specs/src/rationale/message_block_layout.md index 1af44b31e2..2fef2fa9a3 100644 --- a/specs/src/rationale/message_block_layout.md +++ b/specs/src/rationale/message_block_layout.md @@ -39,6 +39,7 @@ As a non-consensus-critical default, we can impose one additional rule on messag > Messages start at an index that is a multiple of the message minimum square size. The message minimum square size is the smallest square that can contain the message in isolation (i.e. a square with only this message and no other transactions or messages). +In the constraint mentioned above, the number of rows/columns in the minimum square size should be a power of 2. With the above constraint, we can compute subtree roots deterministically. In order to compute the subtree roots, split the message into chunks that are of maximum size: message minimum square size. As an example, a message of length `11` has a minimum square size of `4` because `11` is not greater than `4 * 4 = 16` total shares. Split the message into chunks of length `4, 4, 2, 1`. The resulting slices are the leaves of subtrees whose roots can be computed. These subtree roots will be present as internal nodes in the NMT of _some_ row(s). This is similar to [Merkle Mountain Ranges](https://www.usenix.org/legacy/event/sec09/tech/full_papers/crosby.pdf), though with the largest subtree bounded by the message minimum square size rather than being unbounded. From 1547891819ceb8381ed74f0e3dbaba6807137215 Mon Sep 17 00:00:00 2001 From: Rootul P Date: Fri, 3 Mar 2023 15:43:25 -0500 Subject: [PATCH 17/22] docs: replace evidence with PFBs in square diagram (#1436) --- specs/src/rationale/message_block_layout.md | 9 ++- specs/src/specs/consensus.md | 13 +--- specs/src/specs/data_structures.md | 64 ++++--------------- .../specs/figures/block_data_structures.dot | 4 +- .../specs/figures/block_data_structures.svg | 20 ++---- .../figures/rs2d_originaldata_message.svg | 2 +- .../figures/rs2d_originaldata_reserved.svg | 2 +- 7 files changed, 33 insertions(+), 81 deletions(-) diff --git a/specs/src/rationale/message_block_layout.md b/specs/src/rationale/message_block_layout.md index 2fef2fa9a3..2b24b73c8a 100644 --- a/specs/src/rationale/message_block_layout.md +++ b/specs/src/rationale/message_block_layout.md @@ -11,7 +11,14 @@ Celestia uses [a data availability scheme](https://arxiv.org/abs/1809.09044) tha ## Message Layout Rationale -Block data consists of transactions (which modify the Celestia chain's state), intermediate state roots (required for fraud proofs of the aforementioned transactions), messages (binary blobs which do not modify the Celestia state, but which are intended for a Celestia application identified with a provided namespace ID), and other relevant pieces of data (e.g. evidence for slashing). We want to arrange this data into a `k * k` matrix of fixed-sized shares, which will later be committed to in [Namespace Merkle Trees (NMTs)](../specs/data_structures.md#namespace-merkle-tree). +Block data consists of: + +1. Cosmos SDK module transactions (e.g. [MsgSend](https://github.com/cosmos/cosmos-sdk/blob/f71df80e93bffbf7ce5fbd519c6154a2ee9f991b/proto/cosmos/bank/v1beta1/tx.proto#L21-L32)). These modify the Celestia chain's state. +1. Celestia-specific transactions (e.g. [PayForBlobs](../specs/data_structures.md#payforblobdata)). These modify the Celestia chain's state. +1. Intermediate state roots: required for fraud proofs of the aforementioned transactions. +1. Messages: binary blobs which do not modify the Celestia state, but which are intended for a Celestia application identified with a provided namespace ID. + +We want to arrange this data into a `k * k` matrix of fixed-sized shares, which will later be committed to in [Namespace Merkle Trees (NMTs)](../specs/data_structures.md#namespace-merkle-tree). The simplest way we can imagine arranging block data is to simply serialize it all in no particular order, split it into fixed-sized shares, then arrange those shares into the `k * k` matrix in row-major order. However, this naive scheme can be improved in a number of ways, described below. diff --git a/specs/src/specs/consensus.md b/specs/src/specs/consensus.md index 64ce27eb9d..1400d316fe 100644 --- a/specs/src/specs/consensus.md +++ b/specs/src/specs/consensus.md @@ -15,7 +15,6 @@ - [`block.lastCommit`](#blocklastcommit) - [`block.availableData`](#blockavailabledata) - [State Transitions](#state-transitions) - - [`block.availableData.evidenceData`](#blockavailabledataevidencedata) - [`block.availableData.transactionData`](#blockavailabledatatransactiondata) - [SignedTransactionDataTransfer](#signedtransactiondatatransfer) - [SignedTransactionDataMsgPayForData](#signedtransactiondatamsgpayfordata) @@ -100,7 +99,7 @@ TODO The Tendermint consensus protocol is fork-free by construction under an honest majority of stake assumption. -If a block has a [valid commit](#blocklastcommit), it is part of the canonical chain. If equivocation [evidence](./data_structures.md#evidence) is detected for more than 1/3 of voting power, the node must halt. See [rationale doc](../rationale/fork_choice_das.md) for more information. +If a block has a [valid commit](#blocklastcommit), it is part of the canonical chain. If equivocation evidence is detected for more than 1/3 of voting power, the node must halt. ## Block Validity @@ -165,7 +164,7 @@ The block's [available data](./data_structures.md#availabledata) (analogous to t Once parsed, the following checks must be `true`: 1. The commitments of the [erasure-coded extended](./data_structures.md#2d-reed-solomon-encoding-scheme) `availableData` must match those in `header.availableDataHeader`. Implicitly, this means that both rows and columns must be ordered lexicographically by namespace ID since they are committed to in a [Namespace Merkle Tree](data_structures.md#namespace-merkle-tree). -1. Length of `availableData.intermediateStateRootData` == length of `availableData.transactionData` + length of `availableData.evidenceData` + 2. (Two additional state transitions are the [begin](#begin-block) and [end block](#end-block) implicit transitions.) +1. Length of `availableData.intermediateStateRootData` == length of `availableData.transactionData` + length of `availableData.payForBlobData` + 2. (Two additional state transitions are the [begin](#begin-block) and [end block](#end-block) implicit transitions.) ## State Transitions @@ -176,18 +175,12 @@ For this section, the variable `state` represents the [state tree](./data_struct State transitions are applied in the following order: 1. [Begin block](#begin-block). -1. [Evidence](#blockavailabledataevidencedata). 1. [Transactions](#blockavailabledatatransactiondata). 1. [End block](#end-block). -### `block.availableData.evidenceData` - -Evidence is the second set of state transitions that are applied, ahead of [transactions](#blockavailabledatatransactiondata). -Each evidence represents proof of validator misbehavior, and causes a penalty against the validator(s). - ### `block.availableData.transactionData` -Once [evidence has been processed](#blockavailabledataevidencedata), transactions are applied to the state. Note that _transactions_ mutate the state (essentially, the validator set and minimal balances), while _messages_ do not. See [the architecture documentation](./architecture.md) for more info. +Transactions are applied to the state. Note that _transactions_ mutate the state (essentially, the validator set and minimal balances), while _messages_ do not. See [the architecture documentation](./architecture.md) for more info. `block.availableData.transactionData` is simply a list of [WrappedTransaction](./data_structures.md#wrappedtransaction)s. For each wrapped transaction in this list, `wrappedTransaction`, with index `i` (starting from `0`), the following checks must be `true`: diff --git a/specs/src/specs/data_structures.md b/specs/src/specs/data_structures.md index 8800c3bcbd..87d4d92c1c 100644 --- a/specs/src/specs/data_structures.md +++ b/specs/src/specs/data_structures.md @@ -51,13 +51,10 @@ - [SignedTransactionDataBurn](#signedtransactiondataburn) - [SignedTransactionRedelegateCommission](#signedtransactionredelegatecommission) - [SignedTransactionRedelegateReward](#signedtransactionredelegatereward) + - [PayForBlobData](#payforblobdata) - [IntermediateStateRootData](#intermediatestaterootdata) - [WrappedIntermediateStateRoot](#wrappedintermediatestateroot) - [IntermediateStateRoot](#intermediatestateroot) - - [EvidenceData](#evidencedata) - - [Evidence](#evidence) - - [PublicKey](#publickey) - - [Vote](#vote) - [MessageData](#messagedata) - [Message](#message) - [State](#state) @@ -152,7 +149,7 @@ Data that is [erasure-coded](#erasure-coding) for [data availability checks](htt |-----------------------------|---------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------| | `transactionData` | [TransactionData](#transactiondata) | Transaction data. Transactions modify the validator set and balances, and pay fees for messages to be included. | | `intermediateStateRootData` | [IntermediateStateRootData](#intermediatestaterootdata) | Intermediate state roots used for fraud proofs. | -| `evidenceData` | [EvidenceData](#evidencedata) | Evidence used for slashing conditions (e.g. equivocation). | +| `payForBlobData` | [PayForBlobData](#payforblobdata) | PayForBlob data. Transactions that pay for blobs to be included. | | `messageData` | [MessageData](#messagedata) | Message data. Messages are app data. | ### Commit @@ -188,7 +185,7 @@ Abstraction over transaction fees. Address is a [type alias](#type-aliases). -Addresses are the [hash](#hashing) [digest](#hashdigest) of the [public key](#publickey). +Addresses are the [hash](#hashing) [digest](#hashdigest) of the [public key](https://docs.cosmos.network/v0.46/basics/accounts.html#public-keys). Addresses have a length of 32 bytes. @@ -246,7 +243,7 @@ Unless otherwise indicated explicitly, objects are first [serialized](#serializa Consensus-critical data is authenticated using [ECDSA](https://www.secg.org/sec1-v2.pdf), with the curve [secp256k1](https://en.bitcoin.it/wiki/Secp256k1). A highly-optimized library is available in C (), with wrappers in Go () and Rust (). -[Public keys](#publickey) are encoded in uncompressed form, as the concatenation of the `x` and `y` values. No prefix is needed to distinguish between encoding schemes as this is the only encoding supported. +[Public keys](https://docs.cosmos.network/v0.46/basics/accounts.html#public-keys) are encoded in uncompressed form, as the concatenation of the `x` and `y` values. No prefix is needed to distinguish between encoding schemes as this is the only encoding supported. Deterministic signatures ([RFC-6979](https://tools.ietf.org/rfc/rfc6979.txt)) should be used when signing, but this is not enforced at the protocol level as it cannot be. @@ -504,8 +501,8 @@ For shares **with a reserved namespace ID through [`NAMESPACE_ID_MAX_RESERVED`]( - If this is the first share of a sequence, the next [`SEQUENCE_BYTES`](./consensus.md#constants) contain a big endian `uint32` that represents the length of the sequence that follows in bytes. - The next [`SHARE_RESERVED_BYTES`](./consensus.md#constants) bytes are the starting byte of the length of the [canonically serialized](#serialization) first request that starts in the share, or `0` if there is none, as an unsigned [varint](https://developers.google.com/protocol-buffers/docs/encoding). -- The remaining [`SHARE_SIZE`](./consensus.md#constants)`-`[`NAMESPACE_ID_BYTES`](./consensus.md#constants)`-`[`SHARE_INFO_BYTES`](./consensus.md#constants) `-` [`SEQUENCE_BYTES`](.consensus.md#constants) bytes (only if this is the first share of a sequence) `-` [`SHARE_RESERVED_BYTES`](./consensus.md#constants) bytes are transactions, intermediate state roots, or evidence data depending on the namespace of ths share. Each transaction, intermediate state root, or evidence is prefixed with a [varint](https://developers.google.com/protocol-buffers/docs/encoding) of the length of that unit. -- If there is insufficient transaction, intermediate state root, or evidence data to fill the share, the remaining bytes are filled with `0`. +- The remaining [`SHARE_SIZE`](./consensus.md#constants)`-`[`NAMESPACE_ID_BYTES`](./consensus.md#constants)`-`[`SHARE_INFO_BYTES`](./consensus.md#constants) `-` [`SEQUENCE_BYTES`](.consensus.md#constants) bytes (only if this is the first share of a sequence) `-` [`SHARE_RESERVED_BYTES`](./consensus.md#constants) bytes are transactions, intermediate state roots, or PayForBlob transaction data. Each transaction, intermediate state root, or PayForBlob transaction is prefixed with a [varint](https://developers.google.com/protocol-buffers/docs/encoding) of the length of that unit. +- If there is insufficient transaction, intermediate state root, or PayForBlob transaction data to fill the share, the remaining bytes are filled with `0`. First share in a sequence: @@ -567,17 +564,17 @@ The first [`NAMESPACE_ID_BYTES`](./consensus.md#constants) of a share's raw data ### Arranging Available Data Into Shares -The previous sections described how some original data, arranged into a `k * k` matrix, can be extended into a `2k * 2k` matrix and committed to with NMT roots. This section specifies how [available data](#available-data) (which includes [transactions](#transactiondata), [intermediate state roots](#intermediatestaterootdata), [evidence](#evidencedata), and [messages](#messagedata)) is arranged into the matrix in the first place. +The previous sections described how some original data, arranged into a `k * k` matrix, can be extended into a `2k * 2k` matrix and committed to with NMT roots. This section specifies how [available data](#available-data) (which includes [transactions](#transactiondata), [intermediate state roots](#intermediatestaterootdata), PayForBlob transactions, and [messages](#messagedata)) is arranged into the matrix in the first place. Then, -1. For each of `transactionData`, `intermediateStateRootData`, and `evidenceData`, [serialize](#serialization): +1. For each of `transactionData`, `intermediateStateRootData`, PayForBlob transactions, [serialize](#serialization): 1. For each request in the list: 1. [Serialize](#serialization) the request (individually). 1. Compute the length of each serialized request, [serialize the length](#serialization), and pre-pend the serialized request with its serialized length. 1. Split up the length/request pairs into [`SHARE_SIZE`](./consensus.md#constants)`-`[`NAMESPACE_ID_BYTES`](./consensus.md#constants)`-`[`SHARE_RESERVED_BYTES`](./consensus.md#constants)-byte chunks. 1. Create a [share](#share) out of each chunk. This data has a _reserved_ namespace ID, so the first [`NAMESPACE_ID_BYTES`](./consensus.md#constants)`+`[`SHARE_RESERVED_BYTES`](./consensus.md#constants) bytes for these shares must be [set specially](#share). -1. Concatenate the lists of shares in the order: transactions, intermediate state roots, evidence. +1. Concatenate the lists of shares in the order: transactions, intermediate state roots, PayForBlob transactions. Note that by construction, each share only has a single namespace, and that the list of concatenated shares is [lexicographically ordered by namespace ID](consensus.md#reserved-namespace-ids). @@ -797,6 +794,8 @@ Assigns validator's pending commission to a delegation. Adds delegation's pending rewards to voting power. +### PayForBlobData + ### IntermediateStateRootData | name | type | description | @@ -816,47 +815,6 @@ Adds delegation's pending rewards to voting power. |--------|---------------------------|------------------------------------------------------------------------------------------| | `root` | [HashDigest](#hashdigest) | Root of intermediate state, which is composed of the global state and the validator set. | -### EvidenceData - -Wrapper for evidence data. - -| name | type | description | -|-------------|---------------------------|------------------------------------------------| -| `evidences` | [Evidence](#evidence)`[]` | List of evidence used for slashing conditions. | - -#### Evidence - -| name | type | description | -|----------|-------------------------|-------------| -| `pubKey` | [PublicKey](#publickey) | | -| `voteA` | [Vote](#vote) | | -| `voteB` | [Vote](#vote) | | - -#### PublicKey - -| name | type | description | -|------|------------|--------------------------| -| `x` | `byte[32]` | `x` value of public key. | -| `y` | `byte[32]` | `y` value of public key. | - -#### Vote - -```C++ -enum VoteType : uint8_t { - Prevote = 1, - Precommit = 2, -}; -``` - -| name | type | description | -|--------------|---------------------------|-------------| -| `type` | `VoteType` | | -| `height` | [Height](#type-aliases) | | -| `round` | [Round](#type-aliases) | | -| `headerHash` | [HashDigest](#hashdigest) | | -| `timestamp` | [Timestamp](#timestamp) | | -| `signature` | [Signature](#signature) | | - ### MessageData | name | type | description | diff --git a/specs/src/specs/figures/block_data_structures.dot b/specs/src/specs/figures/block_data_structures.dot index f98a4e4c4a..70812f76b7 100644 --- a/specs/src/specs/figures/block_data_structures.dot +++ b/specs/src/specs/figures/block_data_structures.dot @@ -12,7 +12,7 @@ digraph G { subgraph cluster_body { label = "availableData"; - struct3 [label = "{ | { transactionData | intermediateStateRoots | evidenceData | messageData } }"]; + struct3 [label = "{ | { transactionData | intermediateStateRoots | payForBlobData | messageData } }"]; } subgraph cluster_lastcommit { @@ -33,4 +33,4 @@ digraph G { edge [style = invis]; struct1 -> struct3; struct1 -> struct4; -} \ No newline at end of file +} diff --git a/specs/src/specs/figures/block_data_structures.svg b/specs/src/specs/figures/block_data_structures.svg index 5c0bddf8dc..c47a8b80c9 100644 --- a/specs/src/specs/figures/block_data_structures.svg +++ b/specs/src/specs/figures/block_data_structures.svg @@ -1,11 +1,5 @@ - - - - - + G @@ -54,17 +48,17 @@ intermediateStateRoots -evidenceData +payForBlobData messageData -struct4:f0->struct3 +struct4:f0->struct3 NMT roots to -erasure-coded data +erasure-coded data @@ -100,7 +94,7 @@ -struct1:f9->struct4 +struct1:f9->struct4 Merkle root of @@ -109,7 +103,7 @@ -struct1:f5->struct2 +struct1:f5->struct2 diff --git a/specs/src/specs/figures/rs2d_originaldata_message.svg b/specs/src/specs/figures/rs2d_originaldata_message.svg index 23f2065d77..0e1c5dec3f 100644 --- a/specs/src/specs/figures/rs2d_originaldata_message.svg +++ b/specs/src/specs/figures/rs2d_originaldata_message.svg @@ -26,7 +26,7 @@ TXs ISRs - evi. + PFBs msg1 msg1 diff --git a/specs/src/specs/figures/rs2d_originaldata_reserved.svg b/specs/src/specs/figures/rs2d_originaldata_reserved.svg index 5e74465a38..3b64e73edc 100644 --- a/specs/src/specs/figures/rs2d_originaldata_reserved.svg +++ b/specs/src/specs/figures/rs2d_originaldata_reserved.svg @@ -26,5 +26,5 @@ TXs ISRs - evi. + PFBs From c09843d07d4c3842753138de96b304b4866e8f5d Mon Sep 17 00:00:00 2001 From: Rootul P Date: Mon, 6 Mar 2023 10:40:20 -0500 Subject: [PATCH 18/22] specs: auto generate table of contents (#1441) ## Motivation The table of contents at the top of each file can be a maintenance burden. We can auto-generate it with [mdbook-toc](https://github.com/badboy/mdbook-toc). ## Screenshot Before | After --- | --- before | after --- specs/README.md | 7 +- specs/book.toml | 4 ++ specs/src/rationale/message_block_layout.md | 5 +- specs/src/specs/block_proposer.md | 3 +- specs/src/specs/consensus.md | 30 +-------- specs/src/specs/data_structures.md | 73 +-------------------- specs/src/specs/networking.md | 11 +--- 7 files changed, 15 insertions(+), 118 deletions(-) diff --git a/specs/README.md b/specs/README.md index e12959d126..a22e9928e6 100644 --- a/specs/README.md +++ b/specs/README.md @@ -2,7 +2,12 @@ ## Building From Source -Prerequisite: [install mdBook](https://rust-lang.github.io/mdBook/guide/installation.html). +Install [mdbook](https://rust-lang.github.io/mdBook/guide/installation.html) and [mdbook-toc](https://github.com/badboy/mdbook-toc): + +```sh +cargo install mdbook +cargo install mdbook-toc +``` To build book: diff --git a/specs/book.toml b/specs/book.toml index 694ba1019b..bce95681cd 100644 --- a/specs/book.toml +++ b/specs/book.toml @@ -10,3 +10,7 @@ git-repository-url = "https://github.com/celestiaorg/celestia-app" [rust] edition = "2021" + +[preprocessor.toc] +command = "mdbook-toc" +renderer = ["html"] diff --git a/specs/src/rationale/message_block_layout.md b/specs/src/rationale/message_block_layout.md index 2b24b73c8a..684df859e4 100644 --- a/specs/src/rationale/message_block_layout.md +++ b/specs/src/rationale/message_block_layout.md @@ -1,9 +1,6 @@ # Message Layout -- [Preamble](#preamble) -- [Message Layout Rationale](#message-layout-rationale) - - [Non-Interactive Default Rules](#non-interactive-default-rules) - - [Caveats](#caveats) + ## Preamble diff --git a/specs/src/specs/block_proposer.md b/specs/src/specs/block_proposer.md index 948a5ecb78..d4afa7269f 100644 --- a/specs/src/specs/block_proposer.md +++ b/specs/src/specs/block_proposer.md @@ -1,7 +1,6 @@ # Honest Block Proposer -- [Deciding on a Block Size](#deciding-on-a-block-size) -- [Laying out Transactions and Messages](#laying-out-transactions-and-messages) + This document describes the tasks of an honest block proposer to assemble a new block. Performing these actions is not enforced by the [consensus rules](./consensus.md), so long as a valid block is produced. diff --git a/specs/src/specs/consensus.md b/specs/src/specs/consensus.md index 1400d316fe..db45405cc0 100644 --- a/specs/src/specs/consensus.md +++ b/specs/src/specs/consensus.md @@ -1,34 +1,6 @@ # Consensus Rules -- [System Parameters](#system-parameters) - - [Units](#units) - - [Constants](#constants) - - [Reserved Namespace IDs](#reserved-namespace-ids) - - [Reserved State Subtree IDs](#reserved-state-subtree-ids) - - [Rewards and Penalties](#rewards-and-penalties) -- [Leader Selection](#leader-selection) -- [Fork Choice](#fork-choice) -- [Block Validity](#block-validity) -- [Block Structure](#block-structure) - - [`block.header`](#blockheader) - - [`block.availableDataHeader`](#blockavailabledataheader) - - [`block.lastCommit`](#blocklastcommit) - - [`block.availableData`](#blockavailabledata) -- [State Transitions](#state-transitions) - - [`block.availableData.transactionData`](#blockavailabledatatransactiondata) - - [SignedTransactionDataTransfer](#signedtransactiondatatransfer) - - [SignedTransactionDataMsgPayForData](#signedtransactiondatamsgpayfordata) - - [SignedTransactionDataCreateValidator](#signedtransactiondatacreatevalidator) - - [SignedTransactionDataBeginUnbondingValidator](#signedtransactiondatabeginunbondingvalidator) - - [SignedTransactionDataUnbondValidator](#signedtransactiondataunbondvalidator) - - [SignedTransactionDataCreateDelegation](#signedtransactiondatacreatedelegation) - - [SignedTransactionDataBeginUnbondingDelegation](#signedtransactiondatabeginunbondingdelegation) - - [SignedTransactionDataUnbondDelegation](#signedtransactiondataunbonddelegation) - - [SignedTransactionDataBurn](#signedtransactiondataburn) - - [SignedTransactionRedelegateCommission](#signedtransactionredelegatecommission) - - [SignedTransactionRedelegateReward](#signedtransactionredelegatereward) - - [Begin Block](#begin-block) - - [End Block](#end-block) + ## System Parameters diff --git a/specs/src/specs/data_structures.md b/specs/src/specs/data_structures.md index 87d4d92c1c..1e0feddf6f 100644 --- a/specs/src/specs/data_structures.md +++ b/specs/src/specs/data_structures.md @@ -1,77 +1,6 @@ # Data Structures -- [Data Structures Overview](#data-structures-overview) -- [Type Aliases](#type-aliases) -- [Blockchain Data Structures](#blockchain-data-structures) - - [Block](#block) - - [Header](#header) - - [AvailableDataHeader](#availabledataheader) - - [AvailableData](#availabledata) - - [Commit](#commit) - - [Timestamp](#timestamp) - - [HashDigest](#hashdigest) - - [TransactionFee](#transactionfee) - - [Address](#address) - - [CommitSig](#commitsig) - - [Signature](#signature) -- [ConsensusVersion](#consensusversion) -- [Serialization](#serialization) -- [Hashing](#hashing) -- [Public-Key Cryptography](#public-key-cryptography) -- [Merkle Trees](#merkle-trees) - - [Binary Merkle Tree](#binary-merkle-tree) - - [BinaryMerkleTreeInclusionProof](#binarymerkletreeinclusionproof) - - [Namespace Merkle Tree](#namespace-merkle-tree) - - [NamespaceMerkleTreeInclusionProof](#namespacemerkletreeinclusionproof) - - [Sparse Merkle Tree](#sparse-merkle-tree) - - [SparseMerkleTreeInclusionProof](#sparsemerkletreeinclusionproof) -- [Erasure Coding](#erasure-coding) - - [Reed-Solomon Erasure Coding](#reed-solomon-erasure-coding) - - [2D Reed-Solomon Encoding Scheme](#2d-reed-solomon-encoding-scheme) - - [Share](#share) - - [Compact Share](#compact-share) - - [Sparse Share](#sparse-share) - - [Parity Share](#parity-share) - - [Namespace Padding Share](#namespace-padding-share) - - [Tail Padding Share](#tail-padding-share) - - [Arranging Available Data Into Shares](#arranging-available-data-into-shares) -- [Available Data](#available-data) - - [TransactionData](#transactiondata) - - [WrappedTransaction](#wrappedtransaction) - - [Transaction](#transaction) - - [SignedTransactionData](#signedtransactiondata) - - [SignedTransactionDataTransfer](#signedtransactiondatatransfer) - - [SignedTransactionDataMsgPayForData](#signedtransactiondatamsgpayfordata) - - [SignedTransactionDataCreateValidator](#signedtransactiondatacreatevalidator) - - [SignedTransactionDataBeginUnbondingValidator](#signedtransactiondatabeginunbondingvalidator) - - [SignedTransactionDataUnbondValidator](#signedtransactiondataunbondvalidator) - - [SignedTransactionDataCreateDelegation](#signedtransactiondatacreatedelegation) - - [SignedTransactionDataBeginUnbondingDelegation](#signedtransactiondatabeginunbondingdelegation) - - [SignedTransactionDataUnbondDelegation](#signedtransactiondataunbonddelegation) - - [SignedTransactionDataBurn](#signedtransactiondataburn) - - [SignedTransactionRedelegateCommission](#signedtransactionredelegatecommission) - - [SignedTransactionRedelegateReward](#signedtransactionredelegatereward) - - [PayForBlobData](#payforblobdata) - - [IntermediateStateRootData](#intermediatestaterootdata) - - [WrappedIntermediateStateRoot](#wrappedintermediatestateroot) - - [IntermediateStateRoot](#intermediatestateroot) - - [MessageData](#messagedata) - - [Message](#message) -- [State](#state) - - [StateElement](#stateelement) - - [Account](#account) - - [Delegation](#delegation) - - [Validator](#validator) - - [ActiveValidatorCount](#activevalidatorcount) - - [ActiveVotingPower](#activevotingpower) - - [ProposerBlockReward](#proposerblockreward) - - [ProposerInitialVotingPower](#proposerinitialvotingpower) - - [ValidatorQueueHead](#validatorqueuehead) - - [PeriodEntry](#periodentry) - - [Decimal](#decimal) - - [MessagePaid](#messagepaid) - - [MessagePaidHead](#messagepaidhead) -- [Consensus Parameters](#consensus-parameters) + ## Data Structures Overview diff --git a/specs/src/specs/networking.md b/specs/src/specs/networking.md index e7b82c7120..3c455a7455 100644 --- a/specs/src/specs/networking.md +++ b/specs/src/specs/networking.md @@ -1,15 +1,6 @@ # Networking -- [Wire Format](#wire-format) - - [AvailableData](#availabledata) - - [AvailableDataRow](#availabledatarow) - - [ConsensusProposal](#consensusproposal) - - [MsgWirePayForData](#msgwirepayfordata) -- [Invalid Erasure Coding](#invalid-erasure-coding) - - [ShareProof](#shareproof) - - [BadEncodingFraudProof](#badencodingfraudproof) -- [Invalid State Update](#invalid-state-update) - - [StateFraudProof](#statefraudproof) + ## Wire Format From cb2ccdfa01ab7916593650b8963ce2698adf57c1 Mon Sep 17 00:00:00 2001 From: Rootul P Date: Fri, 10 Mar 2023 16:14:00 -0500 Subject: [PATCH 19/22] docs: note about Q3 (#1467) --- specs/src/specs/data_structures.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/specs/src/specs/data_structures.md b/specs/src/specs/data_structures.md index 1e0feddf6f..5f61370881 100644 --- a/specs/src/specs/data_structures.md +++ b/specs/src/specs/data_structures.md @@ -390,6 +390,8 @@ Where `A -> B` indicates that `B` is computed using [erasure coding](#reed-solom - `Q0 -> Q2` for each column in `Q0` and `Q2` - `Q2 -> Q3` for each row in `Q2` and `Q3` +Note that the parity data in `Q3` will be identical if it is vertically extended from `Q1` or horizontally extended from `Q2`. + ![fig: RS2D encoding: extending data.](./figures/rs2d_extending.svg) As an example, the parity data in the second column of `Q2` (in striped purple) is computed by [extending](#reed-solomon-erasure-coding) the original data in the second column of `Q0` (in solid blue). From 3ff81e898de58b46633f2fd93d68465bcf0b2d75 Mon Sep 17 00:00:00 2001 From: Rootul Patel Date: Tue, 14 Mar 2023 18:11:38 -0400 Subject: [PATCH 20/22] lint: fix markdownlint --- specs/src/specs/consensus.md | 2 +- specs/src/specs/data_structures.md | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/specs/src/specs/consensus.md b/specs/src/specs/consensus.md index db45405cc0..21618573ef 100644 --- a/specs/src/specs/consensus.md +++ b/specs/src/specs/consensus.md @@ -235,7 +235,7 @@ function messagePaidListInsert(tx, txid) state.messagePaidHead = txid ``` -We define a helper function to compute [F1 entries](../rationale/distributing_rewards.md): +We define a helper function to compute F1 entries: ```py function compute_new_entry(reward, power) diff --git a/specs/src/specs/data_structures.md b/specs/src/specs/data_structures.md index 5f61370881..fd7d110597 100644 --- a/specs/src/specs/data_structures.md +++ b/specs/src/specs/data_structures.md @@ -432,7 +432,7 @@ For shares **with a reserved namespace ID through [`NAMESPACE_ID_MAX_RESERVED`]( - If this is the first share of a sequence, the next [`SEQUENCE_BYTES`](./consensus.md#constants) contain a big endian `uint32` that represents the length of the sequence that follows in bytes. - The next [`SHARE_RESERVED_BYTES`](./consensus.md#constants) bytes are the starting byte of the length of the [canonically serialized](#serialization) first request that starts in the share, or `0` if there is none, as an unsigned [varint](https://developers.google.com/protocol-buffers/docs/encoding). -- The remaining [`SHARE_SIZE`](./consensus.md#constants)`-`[`NAMESPACE_ID_BYTES`](./consensus.md#constants)`-`[`SHARE_INFO_BYTES`](./consensus.md#constants) `-` [`SEQUENCE_BYTES`](.consensus.md#constants) bytes (only if this is the first share of a sequence) `-` [`SHARE_RESERVED_BYTES`](./consensus.md#constants) bytes are transactions, intermediate state roots, or PayForBlob transaction data. Each transaction, intermediate state root, or PayForBlob transaction is prefixed with a [varint](https://developers.google.com/protocol-buffers/docs/encoding) of the length of that unit. +- The remaining [`SHARE_SIZE`](./consensus.md#constants)`-`[`NAMESPACE_ID_BYTES`](./consensus.md#constants)`-`[`SHARE_INFO_BYTES`](./consensus.md#constants) `-` [`SEQUENCE_BYTES`](./consensus.md#constants) bytes (only if this is the first share of a sequence) `-` [`SHARE_RESERVED_BYTES`](./consensus.md#constants) bytes are transactions, intermediate state roots, or PayForBlob transaction data. Each transaction, intermediate state root, or PayForBlob transaction is prefixed with a [varint](https://developers.google.com/protocol-buffers/docs/encoding) of the length of that unit. - If there is insufficient transaction, intermediate state root, or PayForBlob transaction data to fill the share, the remaining bytes are filled with `0`. First share in a sequence: @@ -454,7 +454,7 @@ For shares **with a namespace ID above [`NAMESPACE_ID_MAX_RESERVED`](./consensus > **Note** The first [`NAMESPACE_ID_BYTES`](./consensus.md#constants) of a share's raw data `rawData` is the namespace ID of that share, `namespaceID`. The next [`SHARE_INFO_BYTES`](./consensus.md#constants) bytes are for share information. - If this is the first share of a sequence, the next [`SEQUENCE_BYTES`](./consensus.md#constants) contain a big endian `uint32` that represents the length of the sequence that follows in bytes. -- The remaining [`SHARE_SIZE`](./consensus.md#constants)`-`[`NAMESPACE_ID_BYTES`](./consensus.md#constants)`-`[`SHARE_INFO_BYTES`](./consensus.md#constants) `-` [`SEQUENCE_BYTES`](.consensus.md#constants) bytes (only if this is the first share of a sequence) bytes are message data. Message data are opaque bytes of data that are included in the block but do not impact the state. In other words, the remaining bytes have no special meaning and are simply used to store data. +- The remaining [`SHARE_SIZE`](./consensus.md#constants)`-`[`NAMESPACE_ID_BYTES`](./consensus.md#constants)`-`[`SHARE_INFO_BYTES`](./consensus.md#constants) `-` [`SEQUENCE_BYTES`](./consensus.md#constants) bytes (only if this is the first share of a sequence) bytes are message data. Message data are opaque bytes of data that are included in the block but do not impact the state. In other words, the remaining bytes have no special meaning and are simply used to store data. - If there is insufficient message data to fill the share, the remaining bytes are filled with `0`. First share in a sequence: @@ -901,8 +901,6 @@ If the queue is empty, `head` is set to the default value (i.e. the hash of the |--------------|-------------------------|---------------------------------------------------------------| | `rewardRate` | [Amount](#type-aliases) | Rewards per unit of voting power accumulated so far, in `1u`. | -For explanation on entries, see the [reward distribution rationale document](../rationale/distributing_rewards.md). - ### Decimal | name | type | description | From b439e04375d816b44a5cdb3d789c4656e8fb8cf4 Mon Sep 17 00:00:00 2001 From: Rootul Patel Date: Wed, 15 Mar 2023 10:28:40 -0400 Subject: [PATCH 21/22] fix: attempt to resolve markdownlint errors Running make lint locally doesn't catch these errors --- specs/src/specs/consensus.md | 2 +- specs/src/specs/networking.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/src/specs/consensus.md b/specs/src/specs/consensus.md index 21618573ef..8b54d61d45 100644 --- a/specs/src/specs/consensus.md +++ b/specs/src/specs/consensus.md @@ -152,7 +152,7 @@ State transitions are applied in the following order: ### `block.availableData.transactionData` -Transactions are applied to the state. Note that _transactions_ mutate the state (essentially, the validator set and minimal balances), while _messages_ do not. See [the architecture documentation](./architecture.md) for more info. +Transactions are applied to the state. Note that _transactions_ mutate the state (essentially, the validator set and minimal balances), while _messages_ do not. `block.availableData.transactionData` is simply a list of [WrappedTransaction](./data_structures.md#wrappedtransaction)s. For each wrapped transaction in this list, `wrappedTransaction`, with index `i` (starting from `0`), the following checks must be `true`: diff --git a/specs/src/specs/networking.md b/specs/src/specs/networking.md index 3c455a7455..5344f3a133 100644 --- a/specs/src/specs/networking.md +++ b/specs/src/specs/networking.md @@ -46,8 +46,8 @@ When receiving a new block proposal `proposal` from the network, the following s 1. The number of elements in `proposal.da_header.row_roots` and `proposal.da_header.row_roots` must be equal. 1. The number of elements in `proposal.da_header.row_roots` must be the same as computed [here](./data_structures.md#header). 1. `proposal.proposer_signature` must be a valid [digital signature](./data_structures.md#public-key-cryptography) over the header hash of `proposal.header` that recovers to `proposal.header.proposer_address`. -1. For [full nodes](./node_types.md#node-type-definitions), `proposal.da_header` must be the result of computing the roots of the shares (received separately). -1. For [light nodes](./node_types.md#node-type-definitions), `proposal.da_header` should be sampled from for availability. +1. For full nodes, `proposal.da_header` must be the result of computing the roots of the shares (received separately). +1. For light nodes, `proposal.da_header` should be sampled from for availability. ### MsgWirePayForData From f2e2e181e74996f89380b24047ed5bf88ac0c0e9 Mon Sep 17 00:00:00 2001 From: Rootul P Date: Thu, 16 Mar 2023 10:37:25 -0400 Subject: [PATCH 22/22] Update .github/workflows/gh-pages.yml Co-authored-by: Matthew Sevey --- .github/workflows/gh-pages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index f53a986d01..324390d9aa 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -21,7 +21,7 @@ jobs: run: mdbook build specs - name: Deploy main - if: github.ref == 'refs/heads/main' && github.event_name == 'push' + if: github.event_name == 'push' uses: peaceiris/actions-gh-pages@v3 with: github_token: ${{ secrets.GITHUB_TOKEN }}