Add a "How to use the CST cursor" How-To (#729)

Closes #334

crates/solidity/outputs/cargo/tests/src/doc_examples/cursor_api.rs

@@ -3,16 +3,59 @@ use semver::Version;
 use slang_solidity::kinds::{FieldName, RuleKind, TokenKind};
 use slang_solidity::language::Language;
 
-const SOURCE: &str = include_str!("cursor_api.sol");
+const SOURCE: &str = include_str!("cursor_api/base.sol");
+
+#[allow(unused_variables)] // snippet below is included as part of the docs
+#[test]
+fn create_cursor() -> Result<()> {
+    // --8<-- [start:create-cursor]
+    let language = Language::new(Version::parse("0.8.0")?)?;
+    let parse_output = language.parse(RuleKind::SourceUnit, SOURCE);
+
+    let cursor = parse_output.create_tree_cursor();
+    // --8<-- [end:create-cursor]
+    Ok(())
+}
+
+#[test]
+fn node_accessors() -> Result<()> {
+    const SOURCE: &str = include_str!("cursor_api/node_accessors.sol");
+    let language = Language::new(Version::parse("0.8.0")?)?;
+    let parse_output = language.parse(RuleKind::SourceUnit, SOURCE);
+
+    let mut cursor = parse_output.create_tree_cursor();
+    // --8<-- [start:example-node-accessors]
+
+    cursor.go_to_next_rule_with_kind(RuleKind::EventParameters);
+
+    let mut parameter_ranges = vec![];
+    let mut cursor = cursor.spawn(); // Only visit children of the first event parameters node
+    while cursor.go_to_next_rule_with_kind(RuleKind::EventParameter) {
+        let text_value = cursor.node().unparse();
+        let range = cursor.text_range();
+        parameter_ranges.push((text_value, range.start.utf8..range.end.utf8));
+    }
+
+    assert_eq!(
+        parameter_ranges,
+        &[
+            ("address indexed src".to_string(), 31..50),
+            (" address indexed dst".to_string(), 51..71),
+            (" uint256 value".to_string(), 72..86)
+        ]
+    );
+    // --8<-- [end:example-node-accessors]
+    Ok(())
+}
 
 #[test]
 fn using_cursor_api() -> Result<()> {
-    // --8<-- [start:example-list-contract-names]
     let language = Language::new(Version::parse("0.8.0")?)?;
     let parse_output = language.parse(RuleKind::SourceUnit, SOURCE);
 
-    let mut contract_names = Vec::new();
     let mut cursor = parse_output.create_tree_cursor();
+    // --8<-- [start:example-list-contract-names]
+    let mut contract_names = Vec::new();
 
     while cursor.go_to_next_rule_with_kinds(&[RuleKind::ContractDefinition]) {
         // You have to make sure you return the cursor to original position
@@ -35,6 +78,7 @@ fn using_spawn() -> Result<()> {
     let language = Language::new(Version::parse("0.8.0")?)?;
     let parse_output = language.parse(RuleKind::SourceUnit, SOURCE);
 
+    // --8<-- [start:example-using-spawn]
     let mut contract_names = Vec::new();
     let mut cursor = parse_output.create_tree_cursor();
 
@@ -51,6 +95,7 @@ fn using_spawn() -> Result<()> {
     }
 
     assert_eq!(contract_names, &["Foo", "Bar", "Baz"]);
+    // --8<-- [end:example-using-spawn]
     Ok(())
 }
 
@@ -59,6 +104,7 @@ fn using_iter() -> Result<()> {
     let language = Language::new(Version::parse("0.8.0")?)?;
     let parse_output = language.parse(RuleKind::SourceUnit, SOURCE);
 
+    // --8<-- [start:example-using-iter]
     let mut contract_names = Vec::new();
     let mut cursor = parse_output.create_tree_cursor();
 
@@ -76,6 +122,7 @@ fn using_iter() -> Result<()> {
     }
 
     assert_eq!(contract_names, &["Foo", "Bar", "Baz"]);
+    // --8<-- [end:example-using-iter]
     Ok(())
 }
 
@@ -103,6 +150,7 @@ fn using_iter_combinators() -> Result<()> {
 
 #[test]
 fn using_iter_with_node_names() -> Result<()> {
+    // --8<-- [start:example-using-cursor-with-names]
     let language = Language::new(Version::parse("0.8.0")?)?;
     let parse_output = language.parse(RuleKind::SourceUnit, SOURCE);
 
@@ -115,5 +163,6 @@ fn using_iter_with_node_names() -> Result<()> {
         .collect();
 
     assert_eq!(names, &["Foo", "Bar", "Baz"]);
+    // --8<-- [end:example-using-cursor-with-names]
     Ok(())
 }

crates/solidity/outputs/cargo/tests/src/doc_examples/cursor_api/node_accessors.sol

@@ -0,0 +1,4 @@
+contract Foo {
+	event Transfer(address indexed src, address indexed dst, uint256 value);
+	event Approval(address indexed owner, address indexed spender, uint256 value);
+}

crates/solidity/outputs/npm/tests/src/doc-examples/cursor-api.ts

@@ -0,0 +1,138 @@
+// --8<-- [start:step-1-imports]
+import { Language } from "@nomicfoundation/slang/language";
+import { FieldName, RuleKind, TokenKind } from "@nomicfoundation/slang/kinds";
+import { NodeType } from "@nomicfoundation/slang/cst";
+
+const source = `
+contract Foo {}
+contract Bar {}
+contract Baz {}
+`;
+
+test("construct the cursor", () => {
+  // --8<-- [start:create-cursor]
+  const language = new Language("0.8.0");
+  const parseTree = language.parse(RuleKind.SourceUnit, source);
+
+  const cursor = parseTree.createTreeCursor();
+  // --8<-- [end:create-cursor]
+  cursor; // Suppress unused warning
+});
+
+test("list contract names", () => {
+  const language = new Language("0.8.0");
+  const parseTree = language.parse(RuleKind.SourceUnit, source);
+
+  const cursor = parseTree.createTreeCursor();
+  // --8<-- [start:example-list-contract-names]
+  const contractNames: string[] = [];
+
+  while (cursor.goToNextRuleWithKinds([RuleKind.ContractDefinition])) {
+    // You have to make sure you return the cursor to original position
+    cursor.goToFirstChild();
+    cursor.goToNextTokenWithKinds([TokenKind.Identifier]);
+
+    const tokenNode = cursor.node();
+    if (tokenNode.type == NodeType.Token) {
+      contractNames.push(tokenNode.text);
+    } else {
+      throw new Error("Identifier should be a token node");
+    }
+
+    cursor.goToParent();
+  }
+
+  expect(contractNames).toEqual(["Foo", "Bar", "Baz"]);
+});
+
+test("use cursor spawn", () => {
+  const language = new Language("0.8.0");
+  const parseTree = language.parse(RuleKind.SourceUnit, source);
+
+  const cursor = parseTree.createTreeCursor();
+  // --8<-- [start:example-using-spawn]
+  const contractNames: string[] = [];
+
+  while (cursor.goToNextRuleWithKinds([RuleKind.ContractDefinition])) {
+    // `.spawn()` creates a new cursor without the path history, which is cheaper
+    // than `.clone()`, which copies the path history.
+    // Do this when you don't want to worry about restoring the position of the
+    // existing cursor.
+    const child_cursor = cursor.spawn();
+    // You have to make sure you return the cursor to original position
+    child_cursor.goToNextTokenWithKinds([TokenKind.Identifier]);
+
+    const tokenNode = child_cursor.node();
+    if (tokenNode.type == NodeType.Token) {
+      contractNames.push(tokenNode.text);
+    } else {
+      throw new Error("Identifier should be a token node");
+    }
+  }
+
+  expect(contractNames).toEqual(["Foo", "Bar", "Baz"]);
+  // --8<-- [end:example-using-spawn]
+});
+
+test("access the node pointed to the cursor", () => {
+  const source = `contract Foo {
+	event Transfer(address indexed src, address indexed dst, uint256 value);
+	event Approval(address indexed owner, address indexed spender, uint256 value);
+}`;
+
+  const language = new Language("0.8.0");
+  const parseTree = language.parse(RuleKind.SourceUnit, source);
+
+  const cursor = parseTree.createTreeCursor();
+  // --8<-- [start:example-node-accessors]
+
+  cursor.goToNextRuleWithKind(RuleKind.EventParameters);
+
+  let parameterRanges: Array<{ textValue: string; range: [number, number] }> = [];
+  // Only visit children of the first event parameters node
+  let cursorChild = cursor.spawn();
+  while (cursorChild.goToNextRuleWithKind(RuleKind.EventParameter)) {
+    // Collect the text value for each parameter rule node
+    let textValue = "";
+    const innerCursor = cursorChild.spawn();
+    while (innerCursor.goToNextToken()) {
+      const node = innerCursor.node();
+      if (node.type != NodeType.Token) continue;
+      textValue += node.text;
+    }
+
+    let range = cursorChild.textRange;
+    parameterRanges.push({ textValue, range: [range.start.utf8, range.end.utf8] });
+  }
+
+  expect(parameterRanges).toEqual([
+    { textValue: "address indexed src", range: [31, 50] },
+    { textValue: " address indexed dst", range: [51, 71] },
+    { textValue: " uint256 value", range: [72, 86] },
+  ]);
+  // --8<-- [end:example-node-accessors]
+});
+
+test("access the node using its name", () => {
+  const language = new Language("0.8.0");
+  const parseTree = language.parse(RuleKind.SourceUnit, source);
+
+  const cursor = parseTree.createTreeCursor();
+  // --8<-- [start:example-using-cursor-with-names]
+  let names: string[] = [];
+
+  while (cursor.goToNextRuleWithKind(RuleKind.ContractDefinition)) {
+    const innerCursor = cursor.spawn();
+    while (innerCursor.goToNext()) {
+      const node = innerCursor.node();
+      const nodeName = innerCursor.nodeName;
+
+      if (node.type == NodeType.Token && nodeName == FieldName.Name) {
+        names.push(node.text);
+      }
+    }
+  }
+
+  expect(names).toEqual(["Foo", "Bar", "Baz"]);
+  // --8<-- [end:example-using-cursor-with-names]
+});

documentation/public/user-guide/npm-package/NAV.md

@@ -1,2 +1,3 @@
 -   [Index](./index.md)
+-   [How to use the CST Cursor](./how-to-use-the-cst-cursor/)
 -   [How to parse a Solidity file](./how-to-parse-a-file/)

documentation/public/user-guide/npm-package/how-to-use-the-cst-cursor/index.md

@@ -0,0 +1,72 @@
+# Using a CST Cursor
+
+The CST (Concrete Syntax Tree) cursor is a powerful tool that allows you to traverse a CST in a depth-first search (DFS) pre-order fashion.
+This guide will walk you through the basics of using a CST cursor in your project.
+
+## Creating a Cursor
+
+First, you need to create an instance of the `Cursor` struct. This is done as follows:
+
+```{ .ts }
+--8<-- "crates/solidity/outputs/npm/tests/src/doc-examples/cursor-api.ts:create-cursor"
+```
+
+## Traversing the CST procedurally
+
+Once you have a cursor, you can use it to traverse the CST in the DFS order. The cursor provides several `goTo*` navigation functions
+for this purpose; each returning `true` if the cursor was successfully moved, and `false` otherwise.
+
+There are three main ways to do it:
+
+-   according to the DFS order, i.e. next/previous nodes,
+-   according to the relationship between the current node and the next node, e.g. siblings/children/parent,
+-   according to the type of the next node, e.g. next token/rule.
+
+As such, the cursor is stateful and keeps track of the path it has taken through the CST.
+It starts at the root it was created at and is completed when it reaches its root when navigating forward.
+
+The below example uses a cursor to collect the names of all contracts in a source file, and returns them as a `Vec<String>`:
+
+```solidity title="input.sol"
+--8<-- "crates/solidity/outputs/cargo/tests/src/doc_examples/cursor_api/base.sol"
+```
+
+```{ .ts }
+--8<-- "crates/solidity/outputs/npm/tests/src/doc-examples/cursor-api.ts:example-list-contract-names"
+```
+
+## Visiting only a sub-tree
+
+Sometimes, it's useful to only visit a sub-tree of the CST. In order to do that, we can use the `Cursor::spawn` function,
+which creates a new cursor that starts at the given node, not copying the previous path history.
+
+```solidity title="input.sol"
+--8<-- "crates/solidity/outputs/cargo/tests/src/doc_examples/cursor_api/base.sol"
+```
+
+```{ .ts }
+--8<-- "crates/solidity/outputs/npm/tests/src/doc-examples/cursor-api.ts:example-using-spawn"
+```
+
+## Accessing the current node
+
+The `Cursor` struct provides several methods that allow you to access the currently visited node, its position in the source code
+and its ancestors.
+
+```solidity title="input.sol"
+--8<-- "crates/solidity/outputs/cargo/tests/src/doc_examples/cursor_api/node_accessors.sol"
+```
+
+```{ .ts }
+--8<-- "crates/solidity/outputs/npm/tests/src/doc-examples/cursor-api.ts:example-node-accessors"
+```
+
+The CST children nodes are often named. Sometimes, it might be more convenient to lookup and access the node by its name, like so:
+
+```solidity title="input.sol"
+--8<-- "crates/solidity/outputs/cargo/tests/src/doc_examples/cursor_api/base.sol"
+```
+
+```{ .ts }
+--8<-- "crates/solidity/outputs/npm/tests/src/doc-examples/cursor-api.ts:example-using-cursor-with-names"
+```

documentation/public/user-guide/rust-crate/NAV.md

@@ -1,2 +1,3 @@
 -   [Index](./index.md)
+-   [How to use the CST Cursor](./how-to-use-the-cst-cursor/)
 -   [How to parse a Solidity file](./how-to-parse-a-file/)

documentation/public/user-guide/rust-crate/how-to-parse-a-file/index.md

@@ -276,7 +276,7 @@ In addition to the `Iterator` implementation, the `Cursor` type also provides pr
 Given the following simple file:
 
 ```{ .solidity}
---8<-- "crates/solidity/outputs/cargo/tests/src/doc_examples/cursor_api.sol"
+--8<-- "crates/solidity/outputs/cargo/tests/src/doc_examples/cursor_api/base.sol"
 ```
 
 We will list the top-level contracts and their names. To do that, we need to visit `ContractDefinition` rule nodes and then their `Identifier` children: