From e917e0b36a1a9fd765ed58f9f9bb8f1a28355fab Mon Sep 17 00:00:00 2001 From: Leo Kettmeir Date: Tue, 8 Oct 2024 03:54:03 -0700 Subject: [PATCH] feat: use only tree-sitter (#644) --- .github/workflows/ci.yml | 12 +- Cargo.lock | 81 +------------- Cargo.toml | 17 +-- deno.json | 2 +- src/html/comrak_adapters.rs | 88 +-------------- src/html/jsdoc.rs | 14 +-- src/html/mod.rs | 11 +- tests/html_test.rs | 6 +- .../html_test__html_doc_files-3.snap | 8 +- .../html_test__html_doc_files-5.snap | 16 +-- .../html_test__html_doc_files-7.snap | 8 +- .../html_test__html_doc_files_rewrite-10.snap | 8 +- .../html_test__html_doc_files_rewrite-12.snap | 20 ++-- .../html_test__html_doc_files_rewrite-14.snap | 8 +- .../html_test__html_doc_files_rewrite-15.snap | 8 +- .../html_test__html_doc_files_rewrite-16.snap | 8 +- .../html_test__html_doc_files_rewrite-17.snap | 8 +- .../html_test__html_doc_files_rewrite-2.snap | 104 ++++++++++-------- .../html_test__html_doc_files_rewrite-3.snap | 8 +- .../html_test__html_doc_files_rewrite-5.snap | 8 +- .../html_test__html_doc_files_rewrite-6.snap | 8 +- .../html_test__html_doc_files_rewrite-7.snap | 10 +- .../html_test__html_doc_files_rewrite-8.snap | 8 +- .../html_test__html_doc_files_rewrite-9.snap | 8 +- .../html_test__html_doc_files_rewrite.snap | 2 +- tests/snapshots/html_test__module_doc.snap | 2 +- tests/snapshots/html_test__symbol_group.snap | 42 +++---- 27 files changed, 176 insertions(+), 347 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3ca46779..0e6e2fbc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,21 +66,11 @@ jobs: - name: Build (Rust) run: cargo build --locked --all-targets - - name: Test (no highlighter) + - name: Test run: | cargo test --locked --all-targets deno test --allow-read --allow-net --allow-env --allow-write - - name: Test (syntect) - run: | - cargo test --locked --all-targets --features=syntect - deno test --allow-read --allow-net --allow-env --allow-write - - - name: Test (tree-sitter) - run: | - cargo test --locked --all-targets --features=tree-sitter - deno test --allow-read --allow-net --allow-env --allow-write - - name: Publish if: | contains(matrix.os, 'ubuntu') && diff --git a/Cargo.lock b/Cargo.lock index 15b7229c..78094ff0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -173,15 +173,6 @@ dependencies = [ "scoped-tls", ] -[[package]] -name = "bincode" -version = "1.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" -dependencies = [ - "serde", -] - [[package]] name = "bitflags" version = "1.3.2" @@ -385,15 +376,6 @@ dependencies = [ "libc", ] -[[package]] -name = "crc32fast" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" -dependencies = [ - "cfg-if", -] - [[package]] name = "criterion" version = "0.4.0" @@ -598,7 +580,6 @@ dependencies = [ "regex", "serde", "serde_json", - "syntect", "termcolor", "tokio", "tree-sitter-bash", @@ -633,9 +614,9 @@ dependencies = [ [[package]] name = "deno_graph" -version = "0.83.1" +version = "0.83.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c62ce152f24a4c0580e7a91431f75de48281157cf645459de8e9d7268dd95b2" +checksum = "e043fe1a12c83bb603f03f56e84ba9f63b1ba240c4e54a10cab76896a1faf65b" dependencies = [ "anyhow", "async-trait", @@ -818,16 +799,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "flate2" -version = "1.0.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0" -dependencies = [ - "crc32fast", - "miniz_oxide", -] - [[package]] name = "fnv" version = "1.0.7" @@ -1351,28 +1322,6 @@ version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" -[[package]] -name = "onig" -version = "6.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c4b31c8722ad9171c6d77d3557db078cab2bd50afcc9d09c8b315c59df8ca4f" -dependencies = [ - "bitflags 1.3.2", - "libc", - "once_cell", - "onig_sys", -] - -[[package]] -name = "onig_sys" -version = "69.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b829e3d7e9cc74c7e315ee8edb185bf4190da5acde74afd7fc59c35b1f086e7" -dependencies = [ - "cc", - "pkg-config", -] - [[package]] name = "oorandom" version = "11.1.4" @@ -1554,12 +1503,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" -[[package]] -name = "pkg-config" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" - [[package]] name = "plotters" version = "0.3.7" @@ -2309,26 +2252,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "syntect" -version = "5.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "874dcfa363995604333cf947ae9f751ca3af4522c60886774c4963943b4746b1" -dependencies = [ - "bincode", - "bitflags 1.3.2", - "flate2", - "fnv", - "once_cell", - "onig", - "regex-syntax", - "serde", - "serde_derive", - "serde_json", - "thiserror", - "walkdir", -] - [[package]] name = "tap" version = "1.0.1" diff --git a/Cargo.toml b/Cargo.toml index 9626c7d2..60daf76a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,14 +41,6 @@ termcolor = "1.4.1" html-escape = { version = "0.2.13", optional = true } comrak = { version = "0.28.0", optional = true, default-features = false } handlebars = { version = "6.1", optional = true, features = ["string_helpers"] } -syntect = { version = "5.2.0", optional = true, default-features = false, features = [ - "parsing", - "default-syntaxes", - "default-themes", - "html", - "dump-load", - "regex-onig", -] } ammonia = { version = "4.0.0", optional = true } tree-sitter-highlight = { version = "0.22.6", optional = true } @@ -75,10 +67,13 @@ pretty_assertions = "1.4.0" insta = { version = "1.39.0", features = ["json"] } [features] -default = ["html", "rust", "ammonia"] +default = ["html", "rust"] rust = [] -html = ["html-escape", "comrak", "handlebars"] -tree-sitter = [ +ammonia = ["dep:ammonia"] +html = [ + "html-escape", + "comrak", + "handlebars", "tree-sitter-highlight", "tree-sitter-javascript", "tree-sitter-typescript", diff --git a/deno.json b/deno.json index 37f1e1a4..2bd4c0b6 100644 --- a/deno.json +++ b/deno.json @@ -4,7 +4,7 @@ "test": "deno test -A", "tailwind": "deno run -A build_css.ts", "gen_html": "deno task tailwind && cargo run --example ddoc --features=html -- --name=Test --html ./tests/testdata/multiple/* --output generated_docs/", - "test:update": "UPDATE=1 cargo test --locked --all-targets && UPDATE=1 cargo test --locked --all-targets --features=tree-sitter && UPDATE=1 cargo test --locked --all-targets --features=syntect && cargo insta test --review" + "test:update": "UPDATE=1 cargo test --locked --all-targets && cargo insta test --accept" }, "workspace": ["js"], "exclude": [ diff --git a/src/html/comrak_adapters.rs b/src/html/comrak_adapters.rs index b6b479a6..79fd9f1d 100644 --- a/src/html/comrak_adapters.rs +++ b/src/html/comrak_adapters.rs @@ -1,8 +1,5 @@ -// Copied and modified from https://github.com/kivikakk/comrak/blob/main/src/plugins/syntect.rs +#![allow(clippy::print_stderr)] -//! Adapter for the Syntect syntax highlighter plugin. - -use crate::html::ShortPath; use comrak::adapters::HeadingAdapter; use comrak::adapters::HeadingMeta; use comrak::adapters::SyntaxHighlighterAdapter; @@ -14,13 +11,7 @@ use std::sync::Arc; use std::sync::Mutex; #[derive(Debug)] -/// Syntect syntax highlighter plugin. pub struct HighlightAdapter { - #[cfg(feature = "syntect")] - pub syntax_set: syntect::parsing::SyntaxSet, - #[cfg(feature = "syntect")] - pub theme_set: syntect::highlighting::ThemeSet, - #[cfg(feature = "tree-sitter")] pub language_cb: fn(&str) -> Option<&'static tree_sitter_highlight::HighlightConfiguration>, pub show_line_numbers: bool, @@ -85,57 +76,6 @@ impl HighlightAdapter { } impl SyntaxHighlighterAdapter for HighlightAdapter { - #[cfg(any( - all(feature = "syntect", not(feature = "tree-sitter")), - all(feature = "syntect", feature = "tree-sitter") - ))] - fn write_highlighted( - &self, - output: &mut dyn Write, - lang: Option<&str>, - code: &str, - ) -> std::io::Result<()> { - let lang = match lang { - Some(l) if !l.is_empty() => l, - _ => "Plain Text", - }; - - let syntax = - self - .syntax_set - .find_syntax_by_token(lang) - .unwrap_or_else(|| { - self - .syntax_set - .find_syntax_by_first_line(code) - .unwrap_or_else(|| self.syntax_set.find_syntax_plain_text()) - }); - - let theme = &self.theme_set.themes["InspiredGitHub"]; - let mut highlighter = syntect::easy::HighlightLines::new(syntax, theme); - - match self.highlight_html( - syntect::util::LinesWithEndings::from(code), - |lines, line| { - let regions = highlighter.highlight_line(line, &self.syntax_set)?; - - syntect::html::append_highlighted_html_for_styled_line( - ®ions, - syntect::html::IncludeBackground::No, - lines, - )?; - - Ok(()) - }, - ) { - Ok(highlighted_code) => output.write_all(highlighted_code.as_bytes())?, - Err(_) => output.write_all(code.as_bytes())?, - } - - self.write_button(output, code) - } - - #[cfg(all(feature = "tree-sitter", not(feature = "syntect")))] fn write_highlighted( &self, output: &mut dyn Write, @@ -163,7 +103,7 @@ impl SyntaxHighlighterAdapter for HighlightAdapter { .unwrap(); output.write_all(html.as_bytes())?; - return self.write_button(output, &code); + return self.write_button(output, code); } Err(err) => { eprintln!("Error rendering code: {}", err); @@ -176,27 +116,7 @@ impl SyntaxHighlighterAdapter for HighlightAdapter { } } comrak::html::escape(output, source)?; - self.write_button(output, &code) - } - - #[cfg(all(not(feature = "syntect"), not(feature = "tree-sitter")))] - fn write_highlighted( - &self, - output: &mut dyn Write, - _lang: Option<&str>, - code: &str, - ) -> std::io::Result<()> { - let code = html_escape::encode_text(code); - let html = self - .highlight_html(code.lines(), |lines, line| { - lines.push_str(&format!("{line}\n")); - - Ok(()) - }) - .unwrap(); - - output.write_all(html.as_bytes())?; - self.write_button(output, &code) + self.write_button(output, code) } fn write_pre_tag( @@ -384,4 +304,4 @@ impl HeadingAdapter for HeadingToCAdapter { #[cfg(feature = "ammonia")] pub type URLRewriter = - Arc, &str) -> String) + Send + Sync>; + Arc, &str) -> String) + Send + Sync>; diff --git a/src/html/jsdoc.rs b/src/html/jsdoc.rs index 0ded0ac0..09427366 100644 --- a/src/html/jsdoc.rs +++ b/src/html/jsdoc.rs @@ -26,8 +26,10 @@ lazy_static! { regex::Regex::new(r"(^\.{0,2}\/)|(^[A-Za-z]+:\S)").unwrap(); static ref MODULE_LINK_RE: regex::Regex = regex::Regex::new(r"^\[(\S+)\](?:\.(\S+)|\s|)$").unwrap(); +} - #[cfg(feature = "ammonia")] +#[cfg(feature = "ammonia")] +lazy_static! { static ref AMMONIA: ammonia::Builder<'static> = { let mut ammonia_builder = ammonia::Builder::default(); @@ -78,19 +80,17 @@ lazy_static! { ], ) .link_rel(Some("nofollow")) - .url_relative( - ammonia::UrlRelative::Custom(Box::new(AmmoniaRelativeUrlEvaluator()))); - - #[cfg(feature = "syntect")] - ammonia_builder.add_tag_attributes("span", ["style"]); + .url_relative(ammonia::UrlRelative::Custom(Box::new( + AmmoniaRelativeUrlEvaluator(), + ))); - #[cfg(feature = "tree-sitter")] ammonia_builder.add_allowed_classes("span", super::tree_sitter::CLASSES); ammonia_builder }; } +#[cfg(feature = "ammonia")] thread_local! { static CURRENT_FILE: RefCell>> = const { RefCell::new(None) }; static URL_REWRITER: RefCell>> = const { RefCell::new(None) }; diff --git a/src/html/mod.rs b/src/html/mod.rs index 9a8df8c5..b34a71ed 100644 --- a/src/html/mod.rs +++ b/src/html/mod.rs @@ -18,8 +18,7 @@ pub mod partition; mod render_context; mod search; mod symbols; -#[cfg(feature = "tree-sitter")] -mod tree_sitter; +pub mod tree_sitter; mod types; mod usage; pub mod util; @@ -793,14 +792,6 @@ pub fn setup_highlighter( show_line_numbers: bool, ) -> comrak_adapters::HighlightAdapter { comrak_adapters::HighlightAdapter { - #[cfg(feature = "syntect")] - syntax_set: syntect::dumps::from_uncompressed_data(include_bytes!( - "./default_newlines.packdump" - )) - .unwrap(), - #[cfg(feature = "syntect")] - theme_set: syntect::highlighting::ThemeSet::load_defaults(), - #[cfg(feature = "tree-sitter")] language_cb: tree_sitter::tree_sitter_language_cb, show_line_numbers, } diff --git a/tests/html_test.rs b/tests/html_test.rs index 2bf4a32a..fbe42623 100644 --- a/tests/html_test.rs +++ b/tests/html_test.rs @@ -182,7 +182,6 @@ async fn html_doc_files() { ] ); - #[cfg(all(not(feature = "syntect"), not(feature = "tree-sitter")))] { insta::assert_snapshot!(files.get("./all_symbols.html").unwrap()); insta::assert_snapshot!(files.get("./index.html").unwrap()); @@ -272,7 +271,6 @@ async fn html_doc_files_rewrite() { ] ); - #[cfg(all(not(feature = "syntect"), not(feature = "tree-sitter")))] { insta::assert_snapshot!(files.get("./all_symbols.html").unwrap()); insta::assert_snapshot!(files.get("./index.html").unwrap()); @@ -386,7 +384,6 @@ async fn symbol_group() { } } - #[cfg(all(not(feature = "syntect"), not(feature = "tree-sitter")))] insta::assert_json_snapshot!(files); } @@ -485,6 +482,5 @@ async fn module_doc() { module_docs.push(module_doc); } - #[cfg(all(not(feature = "syntect"), not(feature = "tree-sitter")))] - insta::assert_json_snapshot!(module_docs); // no syntax + insta::assert_json_snapshot!(module_docs); } diff --git a/tests/snapshots/html_test__html_doc_files-3.snap b/tests/snapshots/html_test__html_doc_files-3.snap index 5bad2540..14958f25 100644 --- a/tests/snapshots/html_test__html_doc_files-3.snap +++ b/tests/snapshots/html_test__html_doc_files-3.snap @@ -52,12 +52,12 @@ expression: "files.get(\"./~/Bar.html\").unwrap()"
-

Usage

import { Bar } from ".";
+    
Usage
import { Bar } from ".";
diff --git a/tests/snapshots/html_test__html_doc_files-5.snap b/tests/snapshots/html_test__html_doc_files-5.snap index fae08311..116ed1f3 100644 --- a/tests/snapshots/html_test__html_doc_files-5.snap +++ b/tests/snapshots/html_test__html_doc_files-5.snap @@ -45,12 +45,12 @@ expression: "files.get(\"./~/Foo.html\").unwrap()"
class Foo - 
using time = new FakeTime();
+          
using time = new FakeTime();
@@ -58,12 +58,12 @@ expression: "files.get(\"./~/Foo.html\").unwrap()"
-

Usage

import { Foo } from ".";
+    
Usage
import { Foo } from ".";
diff --git a/tests/snapshots/html_test__html_doc_files-7.snap b/tests/snapshots/html_test__html_doc_files-7.snap index 0e5e0d01..33950e7d 100644 --- a/tests/snapshots/html_test__html_doc_files-7.snap +++ b/tests/snapshots/html_test__html_doc_files-7.snap @@ -50,12 +50,12 @@ expression: "files.get(\"./~/Foobar.html\").unwrap()"
-

Usage

import Foobar from ".";
+    
Usage
import Foobar from ".";
diff --git a/tests/snapshots/html_test__html_doc_files_rewrite-10.snap b/tests/snapshots/html_test__html_doc_files_rewrite-10.snap index f86f32e3..55fccf94 100644 --- a/tests/snapshots/html_test__html_doc_files_rewrite-10.snap +++ b/tests/snapshots/html_test__html_doc_files_rewrite-10.snap @@ -81,12 +81,12 @@ Type
-

Usage

import { Foo } from ".";
+    
Usage
import { Foo } from ".";
 

 
// This code block is ignored when getting the title of this doc
-const foobar = new Foobar();
+          
// This code block is ignored when getting the title of this doc
+const foobar = new Foobar();

Foobar docs

@@ -80,7 +80,7 @@ const foobar = new Foobar(); -See
@@ -88,12 +88,12 @@ See