diff --git a/.editorconfig b/.editorconfig index d3a8b5b..1f6e352 100644 --- a/.editorconfig +++ b/.editorconfig @@ -2,9 +2,6 @@ root = true [*] charset = utf-8 -end_of_line = lf -insert_final_newline = true -trim_trailing_whitespace = true [*.{json,toml,yml,gyp}] indent_style = space @@ -13,12 +10,13 @@ indent_size = 2 [*.js] indent_style = space indent_size = 2 +quote_type = single -[*.rs] +[*.{c,cc,h}] indent_style = space indent_size = 4 -[*.{c,cc,h}] +[*.rs] indent_style = space indent_size = 4 @@ -37,3 +35,6 @@ indent_size = 8 [Makefile] indent_style = tab indent_size = 8 + +[parser.c] +indent_size = 2 diff --git a/.gitattributes b/.gitattributes index fdd313b..cee6011 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,17 +1,37 @@ -* text eol=lf +* text=auto eol=lf + *.png -text +# Generated source files */src/*.json linguist-generated */src/parser.c linguist-generated */src/tree_sitter/* linguist-generated -*/bindings/** linguist-generated -common/common.mak linguist-generated -bindings/** linguist-generated +# C bindings +*/bindings/c/* linguist-generated + +# Rust bindings +bindings/rust/build.rs linguist-generated +Cargo.lock linguist-generated + +# Node.js bindings +bindings/node/* linguist-generated binding.gyp linguist-generated +package.json linguist-generated +package-lock.json linguist-generated + +# Python bindings +bindings/python/** linguist-generated setup.py linguist-generated -Package.swift linguist-generated +pyproject.toml linguist-generated + +# Go bindings +bindings/go/* linguist-generated +go.mod linguist-generated +go.sum linguist-generated -bindings/rust/lib.rs -linguist-generated -bindings/rust/parser.rs -linguist-generated -bindings/rust/benchmark.rs -linguist-generated +# Swift bindings +/bindings/swift/** linguist-generated +*/bindings/swift/** linguist-generated +Package.swift linguist-generated +Package.resolved linguist-generated diff --git a/contrib/screenshot.png b/.github/screenshot.png similarity index 100% rename from contrib/screenshot.png rename to .github/screenshot.png diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 737dd4e..d275a3a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,8 +9,10 @@ on: - "*/grammar.js" - "*/src/**" - "*/test/**" + - "*/bindings/**" - "bindings/**" - "binding.gyp" + - "setup.py" pull_request: paths: - "scripts/*" @@ -18,8 +20,10 @@ on: - "*/grammar.js" - "*/src/**" - "*/test/**" + - "*/bindings/**" - "bindings/**" - "binding.gyp" + - "setup.py" jobs: test: @@ -34,17 +38,21 @@ jobs: uses: actions/checkout@v4 - name: Install tree-sitter CLI uses: tree-sitter/setup-action/cli@v1 + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{vars.NODE_VERSION}} - name: Build with all extensions - run: npm run build + run: node scripts/build.js env: ALL_EXTENSIONS: 1 - name: Run tests uses: tree-sitter/parser-test-action@v2 with: test-rust: true - test-parser-cmd: npm test + test-parser-cmd: node scripts/test.js - name: Rebuild with default extensions - run: npm run build + run: node scripts/build.js - name: Verify grammar consistency run: git diff --exit-code -- */src/grammar.json fuzz: diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index db1bbd2..540312a 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -5,6 +5,11 @@ on: tags: ["*"] jobs: + github: + uses: tree-sitter/workflows/.github/workflows/release.yml@main + permissions: + contents: write + id-token: write npm: uses: tree-sitter/workflows/.github/workflows/package-npm.yml@main with: diff --git a/.gitignore b/.gitignore index 27fc43f..623310c 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,8 @@ dist/ *.wasm *.obj *.o + +# Moved bindings +bindings/c/ +bindings/swift/TreeSitterMarkdown/ +bindings/swift/TreeSitterMarkdownInline/ diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..d374ae2 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,65 @@ +cmake_minimum_required(VERSION 3.13) + +project(tree-sitter-markdown + VERSION "0.4.0" + DESCRIPTION "Markdown grammar for tree-sitter" + HOMEPAGE_URL "https://github.com/tree-sitter-grammars/tree-sitter-markdown" + LANGUAGES C) + +option(BUILD_SHARED_LIBS "Build using shared libraries" ON) +option(TREE_SITTER_REUSE_ALLOCATOR "Reuse the library allocator" OFF) +option(ALL_EXTENSIONS "Enable all Markdown extensions" OFF) + +set(TREE_SITTER_ABI_VERSION 14 CACHE STRING "Tree-sitter ABI version") +if(NOT ${TREE_SITTER_ABI_VERSION} MATCHES "^[0-9]+$") + unset(TREE_SITTER_ABI_VERSION CACHE) + message(FATAL_ERROR "TREE_SITTER_ABI_VERSION must be an integer") +endif() + +find_program(TREE_SITTER_CLI tree-sitter DOC "Tree-sitter CLI") + +include(GNUInstallDirs) + +macro(add_parser name) + add_custom_command(OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/src/parser.c" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/grammar.js" + COMMAND "${CMAKE_COMMAND}" -E env ALL_EXTENSIONS=$ + -- "${TREE_SITTER_CLI}" generate --abi=${TREE_SITTER_ABI_VERSION} + BYPRODUCTS "${CMAKE_CURRENT_SOURCE_DIR}/src/grammar.json" + "${CMAKE_CURRENT_SOURCE_DIR}/src/node-types.json" + "${CMAKE_CURRENT_SOURCE_DIR}/src/tree_sitter/alloc.h" + "${CMAKE_CURRENT_SOURCE_DIR}/src/tree_sitter/array.h" + "${CMAKE_CURRENT_SOURCE_DIR}/src/tree_sitter/parser.h" + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + COMMENT "Generating parser.c") + + add_library(tree-sitter-${name} + "${CMAKE_CURRENT_SOURCE_DIR}/src/parser.c" + "${CMAKE_CURRENT_SOURCE_DIR}/src/scanner.c") + target_include_directories(tree-sitter-${name} PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src") + + target_compile_definitions(tree-sitter-${name} PRIVATE + $<$:TREE_SITTER_REUSE_ALLOCATOR> + $<$:TREE_SITTER_DEBUG>) + + set_target_properties(tree-sitter-${name} + PROPERTIES + C_STANDARD 11 + POSITION_INDEPENDENT_CODE ON + SOVERSION "${TREE_SITTER_ABI_VERSION}.${PROJECT_VERSION_MAJOR}" + DEFINE_SYMBOL "") + + configure_file("${CMAKE_CURRENT_SOURCE_DIR}/bindings/c/tree-sitter-${name}.pc.in" + "${CMAKE_CURRENT_BINARY_DIR}/tree-sitter-${name}.pc" @ONLY) + + install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/bindings/c/tree-sitter-${name}.h" + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/tree_sitter") + install(FILES "${CMAKE_CURRENT_BINARY_DIR}/tree-sitter-${name}.pc" + DESTINATION "${CMAKE_INSTALL_DATAROOTDIR}/pkgconfig") + install(TARGETS tree-sitter-${name} + LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}") +endmacro() + +add_subdirectory(tree-sitter-markdown tree-sitter-markdown) + +add_subdirectory(tree-sitter-markdown-inline tree-sitter-markdown-inline) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c0d9b59..cfdc86e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -32,8 +32,8 @@ parts of either grammar can be found in the `common` folder. For either of the grammar the most important files are the `grammar.js` which defines most nodes and the `src/scanner.c` which defines nodes that cannot be parsed with normal tree-sitter rules. All other files in the `src` subfolder -are auto-generated by running `tree-sitter generate --no-bindings`. (You need to -install the [tree-sitter cli tool][tree-sitter-cli] first.) +are auto-generated by running `tree-sitter generate`. (You need to install the +[tree-sitter cli tool][tree-sitter-cli] and [Node.js][nodejs] first.) Some syntactical components can be enabled or disabled by environment variables at compile time. The logic for this can be found in the `common/grammar.js` @@ -52,7 +52,7 @@ I will happily accept any pull requests. Before submitting any code please check the following: -* You ran `tree-sitter generate --no-bindings` in the `tree-sitter-markdown` or +* You ran `tree-sitter generate` in the `tree-sitter-markdown` or `tree-sitter-markdown-inline` directories respectively after modifying any `grammar.js` file. * When running `tree-sitter test` only the cases defined in `failing.txt` or @@ -68,8 +68,8 @@ To run the tests, first install the development dependencies (see above), then execute: ```sh -ALL_EXTENSIONS=1 npm run build -npm test +ALL_EXTENSIONS=1 node scripts/build.js +node scripts/test.js ``` [issue]: https://github.com/tree-sitter-grammars/tree-sitter-markdown/issues/new @@ -78,3 +78,4 @@ npm test [commonmark]: https://spec.commonmark.org/ [tree-sitter spec]: https://tree-sitter.github.io/tree-sitter/ [tree-sitter-cli]: https://github.com/tree-sitter/tree-sitter/blob/master/cli/README.md +[nodejs]: https://nodejs.org/ diff --git a/Cargo.toml b/Cargo.toml index bf00c40..2bec52f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,13 +1,13 @@ [package] name = "tree-sitter-md" description = "Markdown grammar for tree-sitter" -version = "0.3.2" +version = "0.4.0" +authors = ["MDeiml"] license = "MIT" readme = "README.md" keywords = ["incremental", "parsing", "tree-sitter", "markdown"] categories = ["parsing", "text-editors"] repository = "https://github.com/tree-sitter-grammars/tree-sitter-markdown" -authors = ["MDeiml"] edition = "2021" autoexamples = false @@ -31,14 +31,14 @@ parser = ["tree-sitter"] path = "bindings/rust/lib.rs" [dependencies] -tree-sitter-language = "0.1.0" -tree-sitter = { version = "0.23", optional = true } +tree-sitter-language = "0.1" +tree-sitter = { version = "0.24", optional = true } [dev-dependencies] -tree-sitter = "0.23.0" +tree-sitter = "0.24.3" [build-dependencies] -cc = "^1.0.89" +cc = "1.1.22" [[bin]] name = "benchmark" diff --git a/Package.resolved b/Package.resolved new file mode 100644 index 0000000..9e0a023 --- /dev/null +++ b/Package.resolved @@ -0,0 +1,16 @@ +{ + "object": { + "pins": [ + { + "package": "SwiftTreeSitter", + "repositoryURL": "https://github.com/ChimeHQ/SwiftTreeSitter", + "state": { + "branch": null, + "revision": "2599e95310b3159641469d8a21baf2d3d200e61f", + "version": "0.8.0" + } + } + ] + }, + "version": 1 +} diff --git a/Package.swift b/Package.swift index dc116f8..375750c 100644 --- a/Package.swift +++ b/Package.swift @@ -1,5 +1,4 @@ // swift-tools-version:5.3 - import PackageDescription let package = Package( @@ -8,38 +7,49 @@ let package = Package( products: [ .library(name: "TreeSitterMarkdown", targets: ["TreeSitterMarkdown", "TreeSitterMarkdownInline"]), ], - dependencies: [], + dependencies: [ + .package(url: "https://github.com/ChimeHQ/SwiftTreeSitter", from: "0.8.0"), + ], targets: [ - .target(name: "TreeSitterMarkdown", - path: "tree-sitter-markdown", - exclude: [ - "test", - "grammar.js", - ], - sources: [ - "src/parser.c", - "src/scanner.c", - ], - resources: [ - .copy("queries") - ], - publicHeadersPath: "bindings/swift", - cSettings: [.headerSearchPath("src")]), - .target(name: "TreeSitterMarkdownInline", - path: "tree-sitter-markdown-inline", - exclude: [ - "test", - "grammar.js", - ], - sources: [ - "src/parser.c", - "src/scanner.c", - ], - resources: [ - .copy("queries") - ], - publicHeadersPath: "bindings/swift", - cSettings: [.headerSearchPath("src")]) + .target( + name: "TreeSitterMarkdown", + path: "tree-sitter-markdown", + sources: [ + "src/parser.c", + "src/scanner.c", + ], + resources: [ + .copy("queries") + ], + publicHeadersPath: "bindings/swift", + cSettings: [.headerSearchPath("src")] + ), + .target( + name: "TreeSitterMarkdownInline", + path: "tree-sitter-markdown-inline", + exclude: [ + "test", + "grammar.js", + ], + sources: [ + "src/parser.c", + "src/scanner.c", + ], + resources: [ + .copy("queries") + ], + publicHeadersPath: "bindings/swift", + cSettings: [.headerSearchPath("src")] + ), + .testTarget( + name: "TreeSitterMarkdownTests", + dependencies: [ + "SwiftTreeSitter", + "TreeSitterMarkdown", + "TreeSitterMarkdownInline", + ], + path: "bindings/swift/TreeSitterMarkdownTests" + ) ], cLanguageStandard: .c11 ) diff --git a/README.md b/README.md index c1817b3..50a243f 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ A Markdown parser for [tree-sitter]. -![screenshot](https://github.com/MDeiml/tree-sitter-markdown/blob/split_parser/contrib/screenshot.png) +![screenshot](https://github.com/MDeiml/tree-sitter-markdown/blob/split_parser/.github/screenshot.png) The parser is designed to read markdown according to the [CommonMark Spec], but some extensions to the spec from different sources such as [Github flavored @@ -19,7 +19,7 @@ For specifics see [Extensions](#extensions) ## Goals Even though this parser has existed for some while and obvious issues are -mostly solved, there are still lots of inaccuarcies in the output. These stem +mostly solved, there are still lots of inaccuracies in the output. These stem from restricting a complex format such as markdown to the quite restricting tree-sitter parsing rules. diff --git a/bindings/.gitignore b/bindings/.gitignore deleted file mode 100644 index a045ec8..0000000 --- a/bindings/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -c/ -swift/*/ diff --git a/bindings/go/binding_test.go b/bindings/go/binding_test.go index 7cf247e..b2381f6 100644 --- a/bindings/go/binding_test.go +++ b/bindings/go/binding_test.go @@ -4,7 +4,7 @@ import ( "testing" tree_sitter "github.com/tree-sitter/go-tree-sitter" - tree_sitter_markdown "github.com/tree-sitter/tree-sitter-markdown/bindings/go" + tree_sitter_markdown "github.com/tree-sitter-grammars/tree-sitter-markdown/bindings/go" ) func TestCanLoadBlockGrammar(t *testing.T) { diff --git a/bindings/node/binding_test.js b/bindings/node/binding_test.js index 67ddfa9..cbba7a6 100644 --- a/bindings/node/binding_test.js +++ b/bindings/node/binding_test.js @@ -3,12 +3,14 @@ const assert = require("node:assert"); const { test } = require("node:test"); +const Parser = require("tree-sitter"); + test("can load block grammar", () => { - const parser = new (require("tree-sitter"))(); + const parser = new Parser(); assert.doesNotThrow(() => parser.setLanguage(require("."))); }); test("can load inline grammar", () => { - const parser = new (require("tree-sitter"))(); + const parser = new Parser(); assert.doesNotThrow(() => parser.setLanguage(require(".").inline)); }); diff --git a/bindings/python/tree_sitter_markdown/__init__.pyi b/bindings/python/tree_sitter_markdown/__init__.pyi index 1e53a6a..0acdc0a 100644 --- a/bindings/python/tree_sitter_markdown/__init__.pyi +++ b/bindings/python/tree_sitter_markdown/__init__.pyi @@ -1,3 +1,3 @@ -def language() -> int: ... +def language() -> object: ... -def inline_language() -> int: ... +def inline_language() -> object: ... diff --git a/bindings/swift/README.md b/bindings/swift/README.md deleted file mode 100644 index 3ff1cd0..0000000 --- a/bindings/swift/README.md +++ /dev/null @@ -1,6 +0,0 @@ -It would be more consistent to put Swift binding C headers here. But, that will not work because of two SPM limitations: - -- Each target must have a unique directory -- C headers must be within that directory - -Building both parsers as one target is possible, but would result in naming issue with the copied queries. If one day either of those limitations is removed, the per-parser bindings can be moved here. diff --git a/bindings/swift/TreeSitterMarkdownTests/TreeSitterMarkdownTests.swift b/bindings/swift/TreeSitterMarkdownTests/TreeSitterMarkdownTests.swift new file mode 100644 index 0000000..65509d7 --- /dev/null +++ b/bindings/swift/TreeSitterMarkdownTests/TreeSitterMarkdownTests.swift @@ -0,0 +1,20 @@ +import XCTest +import SwiftTreeSitter +import TreeSitterMarkdown +import TreeSitterMarkdownInline + +final class TreeSitterXMLTests: XCTestCase { + func testCanLoadBlockGrammar() throws { + let parser = Parser() + let language = Language(language: tree_sitter_markdown()) + XCTAssertNoThrow(try parser.setLanguage(language), + "Error loading Markdown block grammar") + } + + func testCanLoadInlineGrammar() throws { + let parser = Parser() + let language = Language(language: tree_sitter_markdown_inline()) + XCTAssertNoThrow(try parser.setLanguage(language), + "Error loading Markdown inline grammar") + } +} diff --git a/common/common.js b/common/common.js index 0ef7802..95c0f2a 100644 --- a/common/common.js +++ b/common/common.js @@ -1,13 +1,15 @@ -exports.EXTENSION_DEFAULT = !process.env.NO_DEFAULT_EXTENSIONS; -exports.EXTENSION_GFM = process.env.EXTENSION_GFM || exports.EXTENSION_DEFAULT || process.env.ALL_EXTENSIONS; -exports.EXTENSION_TASK_LIST = process.env.EXTENSION_TASK_LIST || exports.EXTENSION_GFM || process.env.ALL_EXTENSIONS; -exports.EXTENSION_STRIKETHROUGH = process.env.EXTENSION_STRIKETHROUGH || exports.EXTENSION_GFM || process.env.ALL_EXTENSIONS; -exports.EXTENSION_PIPE_TABLE = process.env.EXTENSION_PIPE_TABLE || exports.EXTENSION_GFM || process.env.ALL_EXTENSIONS; -exports.EXTENSION_MINUS_METADATA = process.env.EXTENSION_MINUS_METADATA || exports.EXTENSION_DEFAULT || process.env.ALL_EXTENSIONS; -exports.EXTENSION_PLUS_METADATA = process.env.EXTENSION_PLUS_METADATA || exports.EXTENSION_DEFAULT || process.env.ALL_EXTENSIONS; -exports.EXTENSION_TAGS = process.env.EXTENSION_TAGS || process.env.ALL_EXTENSIONS; -exports.EXTENSION_LATEX = process.env.EXTENSION_LATEX || exports.EXTENSION_DEFAULT || process.env.ALL_EXTENSIONS; -exports.EXTENSION_WIKI_LINK = process.env.EXTENSION_WIKI_LINK || process.env.ALL_EXTENSIONS; +/// + +module.exports.EXTENSION_DEFAULT = !process.env.NO_DEFAULT_EXTENSIONS; +module.exports.EXTENSION_GFM = process.env.EXTENSION_GFM || module.exports.EXTENSION_DEFAULT || process.env.ALL_EXTENSIONS; +module.exports.EXTENSION_TASK_LIST = process.env.EXTENSION_TASK_LIST || module.exports.EXTENSION_GFM || process.env.ALL_EXTENSIONS; +module.exports.EXTENSION_STRIKETHROUGH = process.env.EXTENSION_STRIKETHROUGH || module.exports.EXTENSION_GFM || process.env.ALL_EXTENSIONS; +module.exports.EXTENSION_PIPE_TABLE = process.env.EXTENSION_PIPE_TABLE || module.exports.EXTENSION_GFM || process.env.ALL_EXTENSIONS; +module.exports.EXTENSION_MINUS_METADATA = process.env.EXTENSION_MINUS_METADATA || module.exports.EXTENSION_DEFAULT || process.env.ALL_EXTENSIONS; +module.exports.EXTENSION_PLUS_METADATA = process.env.EXTENSION_PLUS_METADATA || module.exports.EXTENSION_DEFAULT || process.env.ALL_EXTENSIONS; +module.exports.EXTENSION_TAGS = process.env.EXTENSION_TAGS || process.env.ALL_EXTENSIONS; +module.exports.EXTENSION_LATEX = process.env.EXTENSION_LATEX || module.exports.EXTENSION_DEFAULT || process.env.ALL_EXTENSIONS; +module.exports.EXTENSION_WIKI_LINK = process.env.EXTENSION_WIKI_LINK || process.env.ALL_EXTENSIONS; const PUNCTUATION_CHARACTERS_REGEX = '!-/:-@\\[-`\\{-~'; const PUNCTUATION_CHARACTERS_ARRAY = [ @@ -17,11 +19,12 @@ const PUNCTUATION_CHARACTERS_ARRAY = [ const PRECEDENCE_LEVEL_LINK = 10; -exports.PRECEDENCE_LEVEL_LINK = PRECEDENCE_LEVEL_LINK; +module.exports.PRECEDENCE_LEVEL_LINK = PRECEDENCE_LEVEL_LINK; -exports.PUNCTUATION_CHARACTERS_REGEX = PUNCTUATION_CHARACTERS_REGEX; +module.exports.PUNCTUATION_CHARACTERS_REGEX = PUNCTUATION_CHARACTERS_REGEX; -exports.rules = { +/** @type {Record) => RuleOrLiteral>} */ +module.exports.rules = { // A backslash escape. This can often be part of different nodes like link labels // @@ -115,7 +118,7 @@ function punctuation_without($, chars) { return seq(choice(...PUNCTUATION_CHARACTERS_ARRAY.filter(c => !chars.includes(c))), optional($._last_token_punctuation)); } -exports.punctuation_without = punctuation_without; +module.exports.punctuation_without = punctuation_without; // Constructs a regex that matches all html entity references. function html_entity_regex() { diff --git a/common/common.mak b/common/common.mak index d3fdeb6..b59a443 100644 --- a/common/common.mak +++ b/common/common.mak @@ -2,21 +2,12 @@ ifeq ($(OS),Windows_NT) $(error Windows is not supported) endif -VERSION := 0.3.2 +HOMEPAGE_URL := https://github.com/tree-sitter-grammars/tree-sitter-markdown +VERSION := 0.4.0 # repository SRC_DIR := src -PARSER_REPO_URL := $(shell git -C $(SRC_DIR) remote get-url origin 2>/dev/null) - -ifeq ($(PARSER_URL),) - PARSER_URL := $(subst .git,,$(PARSER_REPO_URL)) -ifeq ($(shell echo $(PARSER_URL) | grep '^[a-z][-+.0-9a-z]*://'),) - PARSER_URL := $(subst :,/,$(PARSER_URL)) - PARSER_URL := $(subst git@,https://,$(PARSER_URL)) -endif -endif - TS ?= tree-sitter # install directory layout @@ -31,37 +22,30 @@ EXTRAS := $(filter-out $(PARSER),$(wildcard $(SRC_DIR)/*.c)) OBJS := $(patsubst %.c,%.o,$(PARSER) $(EXTRAS)) # flags -ARFLAGS := rcs +ARFLAGS ?= rcs override CFLAGS += -I$(SRC_DIR) -std=c11 -fPIC # ABI versioning -SONAME_MAJOR := $(word 1,$(subst ., ,$(VERSION))) -SONAME_MINOR := $(shell sed -n 's/#define LANGUAGE_VERSION //p' $(PARSER)) +SONAME_MAJOR = $(shell sed -n 's/\#define LANGUAGE_VERSION //p' $(PARSER)) +SONAME_MINOR = $(word 1,$(subst ., ,$(VERSION))) # OS-specific bits ifeq ($(shell uname),Darwin) SOEXT = dylib SOEXTVER_MAJOR = $(SONAME_MAJOR).$(SOEXT) SOEXTVER = $(SONAME_MAJOR).$(SONAME_MINOR).$(SOEXT) - LINKSHARED := $(LINKSHARED)-dynamiclib -Wl, - ifneq ($(ADDITIONAL_LIBS),) - LINKSHARED := $(LINKSHARED)$(ADDITIONAL_LIBS), - endif - LINKSHARED := $(LINKSHARED)-install_name,$(LIBDIR)/lib$(LANGUAGE_NAME).$(SOEXTVER),-rpath,@executable_path/../Frameworks + LINKSHARED = -dynamiclib -Wl,-install_name,$(LIBDIR)/lib$(LANGUAGE_NAME).$(SOEXTVER),-rpath,@executable_path/../Frameworks else SOEXT = so SOEXTVER_MAJOR = $(SOEXT).$(SONAME_MAJOR) SOEXTVER = $(SOEXT).$(SONAME_MAJOR).$(SONAME_MINOR) - LINKSHARED := $(LINKSHARED)-shared -Wl, - ifneq ($(ADDITIONAL_LIBS),) - LINKSHARED := $(LINKSHARED)$(ADDITIONAL_LIBS) - endif - LINKSHARED := $(LINKSHARED)-soname,lib$(LANGUAGE_NAME).$(SOEXTVER) + LINKSHARED = -shared -Wl,-soname,lib$(LANGUAGE_NAME).$(SOEXTVER) endif ifneq ($(filter $(shell uname),FreeBSD NetBSD DragonFly),) - PCLIBDIR := $(PREFIX)/libdata/pkgconfig + PCLIBDIR := $(PREFIX)/libdata/pkgconfig endif + all: lib$(LANGUAGE_NAME).a lib$(LANGUAGE_NAME).$(SOEXT) $(LANGUAGE_NAME).pc lib$(LANGUAGE_NAME).a: $(OBJS) @@ -74,17 +58,16 @@ ifneq ($(STRIP),) endif $(LANGUAGE_NAME).pc: bindings/c/$(LANGUAGE_NAME).pc.in - sed -e 's|@URL@|$(PARSER_URL)|' \ - -e 's|@VERSION@|$(VERSION)|' \ - -e 's|@LIBDIR@|$(LIBDIR)|' \ - -e 's|@INCLUDEDIR@|$(INCLUDEDIR)|' \ - -e 's|@REQUIRES@|$(REQUIRES)|' \ - -e 's|@ADDITIONAL_LIBS@|$(ADDITIONAL_LIBS)|' \ - -e 's|=$(PREFIX)|=$${prefix}|' \ - -e 's|@PREFIX@|$(PREFIX)|' $< > $@ - -$(PARSER): $(SRC_DIR)/grammar.json - $(TS) generate --no-bindings $^ + sed -e 's|@CMAKE_PROJECT_VERSION@|$(VERSION)|' \ + -e 's|@CMAKE_INSTALL_LIBDIR@|$(LIBDIR:$(PREFIX)/%=%)|' \ + -e 's|@CMAKE_INSTALL_INCLUDEDIR@|$(INCLUDEDIR:$(PREFIX)/%=%)|' \ + -e 's|@PROJECT_DESCRIPTION@|$(DESCRIPTION)|' \ + -e 's|@CMAKE_PROJECT_HOMEPAGE_URL@|$(HOMEPAGE_URL)|' \ + -e 's|@CMAKE_INSTALL_PREFIX@|$(PREFIX)|' \ + -e 's|@TS_REQUIRES@|$(REQUIRES)|' $< > $@ + +$(PARSER): $(SRC_DIR)/grammar.js + $(TS) generate $^ install: all install -Dm644 bindings/c/$(LANGUAGE_NAME).h '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter/$(LANGUAGE_NAME).h diff --git a/go.mod b/go.mod index 1408817..9ff2f6b 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,9 @@ -module github.com/tree-sitter/tree-sitter-markdown +module github.com/tree-sitter-grammars/tree-sitter-markdown go 1.23 +toolchain go1.23.0 + require github.com/tree-sitter/go-tree-sitter v0.23.1 require github.com/mattn/go-pointer v0.0.1 // indirect diff --git a/package-lock.json b/package-lock.json index ccb6ebd..3985430 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,21 +1,21 @@ { "name": "@tree-sitter-grammars/tree-sitter-markdown", - "version": "0.3.2", + "version": "0.4.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@tree-sitter-grammars/tree-sitter-markdown", - "version": "0.3.2", + "version": "0.4.0", "hasInstallScript": true, "license": "MIT", "dependencies": { - "node-addon-api": "^8.1.0", - "node-gyp-build": "^4.8.1" + "node-addon-api": "^8.2.1", + "node-gyp-build": "^4.8.2" }, "devDependencies": { "prebuildify": "^6.0.1", - "tree-sitter-cli": "^0.23.0" + "tree-sitter-cli": "^0.24.3" }, "peerDependencies": { "tree-sitter": "^0.21.1" @@ -168,9 +168,10 @@ } }, "node_modules/node-addon-api": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.1.0.tgz", - "integrity": "sha512-yBY+qqWSv3dWKGODD6OGE6GnTX7Q2r+4+DfpqxHSHh8x0B4EKP9+wVGLS6U/AM1vxSNNmUEuIV5EGhYwPpfOwQ==", + "version": "8.2.1", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.2.1.tgz", + "integrity": "sha512-vmEOvxwiH8tlOcv4SyE8RH34rI5/nWVaigUeAUPawC6f0+HoDthwI0vkMu4tbtsZrXq6QXFfrkhjofzKEs5tpA==", + "license": "MIT", "engines": { "node": "^18 || ^20 || >= 21" } @@ -340,9 +341,9 @@ } }, "node_modules/tree-sitter-cli": { - "version": "0.23.0", - "resolved": "https://registry.npmjs.org/tree-sitter-cli/-/tree-sitter-cli-0.23.0.tgz", - "integrity": "sha512-/DdQaPCCOrOYGp9FxGdhFUnHIrjhfbYatQXgNIcmaAOpPunpnDj2vsO/H+svsfQLaFsQ1C+BjgPhpbV28zka1w==", + "version": "0.24.3", + "resolved": "https://registry.npmjs.org/tree-sitter-cli/-/tree-sitter-cli-0.24.3.tgz", + "integrity": "sha512-5vS0SiJf31tMTn9CYLsu5l18qXaw5MLFka3cuGxOB5f4TtgoUSK1Sog6rKmqBc7PvFJq37YcQBjj9giNy2cJPw==", "dev": true, "hasInstallScript": true, "license": "MIT", diff --git a/package.json b/package.json index 7a3023a..04606ef 100644 --- a/package.json +++ b/package.json @@ -1,12 +1,14 @@ { "name": "@tree-sitter-grammars/tree-sitter-markdown", - "version": "0.3.2", + "version": "0.4.0", "description": "Markdown grammar for tree-sitter", "repository": "github:tree-sitter-grammars/tree-sitter-markdown", - "author": "MDeiml (https://github.com/MDeiml)", "license": "MIT", "main": "bindings/node", "types": "bindings/node", + "author": { + "name": "MDeiml" + }, "keywords": [ "incremental", "parsing", @@ -24,12 +26,12 @@ "common/html_entities.json" ], "dependencies": { - "node-addon-api": "^8.1.0", - "node-gyp-build": "^4.8.1" + "node-addon-api": "^8.2.1", + "node-gyp-build": "^4.8.2" }, "devDependencies": { "prebuildify": "^6.0.1", - "tree-sitter-cli": "^0.23.0" + "tree-sitter-cli": "^0.24.3" }, "peerDependencies": { "tree-sitter": "^0.21.1" @@ -40,36 +42,13 @@ } }, "scripts": { - "test": "node scripts/test.js", "build": "node scripts/build.js", "install": "node-gyp-build", - "prebuildify": "prebuildify --napi --strip" + "prestart": "tree-sitter build --wasm", + "start": "tree-sitter playground", + "test": "node --test bindings/node/*_test.js" }, "publishConfig": { "access": "public" - }, - "tree-sitter": [ - { - "scope": "text.markdown", - "path": "tree-sitter-markdown", - "injection-regex": "^(markdown|md)$", - "file-types": [ - "md" - ], - "highlights": "tree-sitter-markdown/queries/highlights.scm", - "injections": "tree-sitter-markdown/queries/injections.scm", - "external-files": [ - "common/common.js" - ] - }, - { - "scope": "text.markdown_inline", - "path": "tree-sitter-markdown-inline", - "highlights": "tree-sitter-markdown-inline/queries/highlights.scm", - "injections": "tree-sitter-markdown-inline/queries/injections.scm", - "external-files": [ - "common/common.js" - ] - } - ] + } } diff --git a/pyproject.toml b/pyproject.toml index a6c3fe3..f27c498 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "tree-sitter-markdown" description = "Markdown grammar for tree-sitter" -version = "0.3.2" +version = "0.4.0" keywords = ["incremental", "parsing", "tree-sitter", "markdown"] classifiers = [ "Intended Audience :: Developers", diff --git a/scripts/build.js b/scripts/build.js index b970fd0..16e450c 100755 --- a/scripts/build.js +++ b/scripts/build.js @@ -5,7 +5,7 @@ const { join } = require("path"); for (const dir of ["tree-sitter-markdown", "tree-sitter-markdown-inline"]) { console.log(`building ${dir}`); - execSync("tree-sitter generate --no-bindings", { + execSync("tree-sitter generate", { stdio: "inherit", cwd: join(__dirname, "..", dir) }); diff --git a/tree-sitter-markdown-inline/CMakeLists.txt b/tree-sitter-markdown-inline/CMakeLists.txt new file mode 100644 index 0000000..5c6b781 --- /dev/null +++ b/tree-sitter-markdown-inline/CMakeLists.txt @@ -0,0 +1,8 @@ +set(PROJECT_DESCRIPTION "Markdown inline grammar for tree-sitter") + +add_parser(markdown-inline) + +add_custom_target(test-inline "${TREE_SITTER_CLI}" test + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/parser.c" + COMMENT "tree-sitter test") diff --git a/tree-sitter-markdown-inline/bindings/c/tree-sitter-markdown-inline.pc.in b/tree-sitter-markdown-inline/bindings/c/tree-sitter-markdown-inline.pc.in index 3302561..ae0ed20 100644 --- a/tree-sitter-markdown-inline/bindings/c/tree-sitter-markdown-inline.pc.in +++ b/tree-sitter-markdown-inline/bindings/c/tree-sitter-markdown-inline.pc.in @@ -1,11 +1,11 @@ -prefix=@PREFIX@ -libdir=@LIBDIR@ -includedir=@INCLUDEDIR@ +prefix=@CMAKE_INSTALL_PREFIX@ +libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@ +includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ Name: tree-sitter-markdown-inline -Description: Markdown grammar for tree-sitter (inline) -URL: @URL@ -Version: @VERSION@ -Requires: @REQUIRES@ -Libs: -L${libdir} @ADDITIONAL_LIBS@ -ltree-sitter-markdown-inline +Description: @PROJECT_DESCRIPTION@ +URL: @PROJECT_HOMEPAGE_URL@ +Version: @PROJECT_VERSION@ +Requires: @TS_REQUIRES@ +Libs: -L${libdir} -ltree-sitter-markdown-inline Cflags: -I${includedir} diff --git a/tree-sitter-markdown-inline/package.json b/tree-sitter-markdown-inline/package.json index 3bcad99..0f6b642 100644 --- a/tree-sitter-markdown-inline/package.json +++ b/tree-sitter-markdown-inline/package.json @@ -1,3 +1,8 @@ { - "main": "../bindings/node/inline.js" + "main": "../bindings/node/inline.js", + "private": true, + "scripts": { + "build": "tree-sitter generate", + "test": "tree-sitter test" + } } diff --git a/tree-sitter-markdown-inline/src/node-types.json b/tree-sitter-markdown-inline/src/node-types.json index a249310..1d056e8 100644 --- a/tree-sitter-markdown-inline/src/node-types.json +++ b/tree-sitter-markdown-inline/src/node-types.json @@ -255,6 +255,7 @@ { "type": "inline", "named": true, + "root": true, "fields": {}, "children": { "multiple": true, diff --git a/tree-sitter-markdown-inline/src/tree_sitter/alloc.h b/tree-sitter-markdown-inline/src/tree_sitter/alloc.h index 1f4466d..1abdd12 100644 --- a/tree-sitter-markdown-inline/src/tree_sitter/alloc.h +++ b/tree-sitter-markdown-inline/src/tree_sitter/alloc.h @@ -12,10 +12,10 @@ extern "C" { // Allow clients to override allocation functions #ifdef TREE_SITTER_REUSE_ALLOCATOR -extern void *(*ts_current_malloc)(size_t); -extern void *(*ts_current_calloc)(size_t, size_t); -extern void *(*ts_current_realloc)(void *, size_t); -extern void (*ts_current_free)(void *); +extern void *(*ts_current_malloc)(size_t size); +extern void *(*ts_current_calloc)(size_t count, size_t size); +extern void *(*ts_current_realloc)(void *ptr, size_t size); +extern void (*ts_current_free)(void *ptr); #ifndef ts_malloc #define ts_malloc ts_current_malloc diff --git a/tree-sitter-markdown/CMakeLists.txt b/tree-sitter-markdown/CMakeLists.txt new file mode 100644 index 0000000..c21cb12 --- /dev/null +++ b/tree-sitter-markdown/CMakeLists.txt @@ -0,0 +1,9 @@ +set(PROJECT_DESCRIPTION "Markdown block grammar for tree-sitter") +set(TS_REQUIRES tree-sitter-markdown-inline) + +add_parser(markdown) + +add_custom_target(test "${TREE_SITTER_CLI}" test + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/parser.c" + COMMENT "tree-sitter test") diff --git a/tree-sitter-markdown/bindings/c/tree-sitter-markdown.pc.in b/tree-sitter-markdown/bindings/c/tree-sitter-markdown.pc.in index d249d6e..0b4d9b3 100644 --- a/tree-sitter-markdown/bindings/c/tree-sitter-markdown.pc.in +++ b/tree-sitter-markdown/bindings/c/tree-sitter-markdown.pc.in @@ -1,11 +1,11 @@ -prefix=@PREFIX@ -libdir=@LIBDIR@ -includedir=@INCLUDEDIR@ +prefix=@CMAKE_INSTALL_PREFIX@ +libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@ +includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ Name: tree-sitter-markdown -Description: Markdown grammar for tree-sitter (block) -URL: @URL@ -Version: @VERSION@ -Requires: @REQUIRES@ -Libs: -L${libdir} @ADDITIONAL_LIBS@ -ltree-sitter-markdown +Description: @PROJECT_DESCRIPTION@ +URL: @PROJECT_HOMEPAGE_URL@ +Version: @PROJECT_VERSION@ +Requires: @TS_REQUIRES@ +Libs: -L${libdir} -ltree-sitter-markdown Cflags: -I${includedir} diff --git a/tree-sitter-markdown/package.json b/tree-sitter-markdown/package.json index b245787..991f821 100644 --- a/tree-sitter-markdown/package.json +++ b/tree-sitter-markdown/package.json @@ -1,3 +1,8 @@ { - "main": "../bindings/node/index.js" + "main": "../bindings/node/index.js", + "private": true, + "scripts": { + "build": "tree-sitter generate", + "test": "tree-sitter test" + } } diff --git a/tree-sitter-markdown/src/parser.c b/tree-sitter-markdown/src/parser.c index 29fa974..69506ed 100644 --- a/tree-sitter-markdown/src/parser.c +++ b/tree-sitter-markdown/src/parser.c @@ -15,9 +15,9 @@ #define LANGUAGE_VERSION 14 #define STATE_COUNT 925 #define LARGE_STATE_COUNT 351 -#define SYMBOL_COUNT 204 +#define SYMBOL_COUNT 203 #define ALIAS_COUNT 4 -#define TOKEN_COUNT 91 +#define TOKEN_COUNT 90 #define EXTERNAL_TOKEN_COUNT 47 #define FIELD_COUNT 1 #define MAX_ALIAS_SEQUENCE_LENGTH 13 @@ -59,178 +59,177 @@ enum ts_symbol_identifiers { anon_sym_TILDE = 33, anon_sym_LPAREN = 34, anon_sym_RPAREN = 35, - sym__newline_token = 36, - anon_sym_DASH_DASH_GT = 37, - anon_sym_QMARK_GT = 38, - anon_sym_RBRACK_RBRACK_GT = 39, - aux_sym__word_token1 = 40, - aux_sym__word_token2 = 41, - aux_sym__word_token3 = 42, - sym__whitespace = 43, - sym__line_ending = 44, - sym__soft_line_ending = 45, - sym__block_close = 46, - sym_block_continuation = 47, - sym__block_quote_start = 48, - sym__indented_chunk_start = 49, - sym_atx_h1_marker = 50, - sym_atx_h2_marker = 51, - sym_atx_h3_marker = 52, - sym_atx_h4_marker = 53, - sym_atx_h5_marker = 54, - sym_atx_h6_marker = 55, - sym_setext_h1_underline = 56, - sym_setext_h2_underline = 57, - sym__thematic_break = 58, - sym__list_marker_minus = 59, - sym__list_marker_plus = 60, - sym__list_marker_star = 61, - sym__list_marker_parenthesis = 62, - sym__list_marker_dot = 63, - sym__list_marker_minus_dont_interrupt = 64, - sym__list_marker_plus_dont_interrupt = 65, - sym__list_marker_star_dont_interrupt = 66, - sym__list_marker_parenthesis_dont_interrupt = 67, - sym__list_marker_dot_dont_interrupt = 68, - sym__fenced_code_block_start_backtick = 69, - sym__fenced_code_block_start_tilde = 70, - sym__blank_line_start = 71, - sym__fenced_code_block_end_backtick = 72, - sym__fenced_code_block_end_tilde = 73, - sym__html_block_1_start = 74, - sym__html_block_1_end = 75, - sym__html_block_2_start = 76, - sym__html_block_3_start = 77, - sym__html_block_4_start = 78, - sym__html_block_5_start = 79, - sym__html_block_6_start = 80, - sym__html_block_7_start = 81, - sym__close_block = 82, - sym__no_indented_chunk = 83, - sym__error = 84, - sym__trigger_error = 85, - sym__eof = 86, - sym_minus_metadata = 87, - sym_plus_metadata = 88, - sym__pipe_table_start = 89, - sym__pipe_table_line_ending = 90, - sym_document = 91, - sym_backslash_escape = 92, - sym_link_label = 93, - sym_link_destination = 94, - sym__link_destination_parenthesis = 95, - sym__text_no_angle = 96, - sym_link_title = 97, - sym__last_token_punctuation = 98, - sym__block = 99, - sym__block_not_section = 100, - sym_section = 101, - sym__section1 = 102, - sym__section2 = 103, - sym__section3 = 104, - sym__section4 = 105, - sym__section5 = 106, - sym__section6 = 107, - sym_thematic_break = 108, - sym__atx_heading1 = 109, - sym__atx_heading2 = 110, - sym__atx_heading3 = 111, - sym__atx_heading4 = 112, - sym__atx_heading5 = 113, - sym__atx_heading6 = 114, - sym__atx_heading_content = 115, - sym__setext_heading1 = 116, - sym__setext_heading2 = 117, - sym_indented_code_block = 118, - sym__indented_chunk = 119, - sym_fenced_code_block = 120, - sym_code_fence_content = 121, - sym_info_string = 122, - sym_language = 123, - sym_html_block = 124, - sym__html_block_1 = 125, - sym__html_block_2 = 126, - sym__html_block_3 = 127, - sym__html_block_4 = 128, - sym__html_block_5 = 129, - sym__html_block_6 = 130, - sym__html_block_7 = 131, - sym_link_reference_definition = 132, - sym__text_inline_no_link = 133, - sym_paragraph = 134, - sym__blank_line = 135, - sym_block_quote = 136, - sym_list = 137, - sym__list_plus = 138, - sym__list_minus = 139, - sym__list_star = 140, - sym__list_dot = 141, - sym__list_parenthesis = 142, - sym_list_marker_plus = 143, - sym_list_marker_minus = 144, - sym_list_marker_star = 145, - sym_list_marker_dot = 146, - sym_list_marker_parenthesis = 147, - sym__list_item_plus = 148, - sym__list_item_minus = 149, - sym__list_item_star = 150, - sym__list_item_dot = 151, - sym__list_item_parenthesis = 152, - sym__list_item_content = 153, - sym__newline = 154, - sym__soft_line_break = 155, - sym__line = 156, - sym__word = 157, - sym_task_list_marker_checked = 158, - sym_task_list_marker_unchecked = 159, - sym_pipe_table = 160, - sym__pipe_table_newline = 161, - sym_pipe_table_delimiter_row = 162, - sym_pipe_table_delimiter_cell = 163, - sym_pipe_table_row = 164, - sym_pipe_table_cell = 165, - aux_sym_document_repeat1 = 166, - aux_sym_document_repeat2 = 167, - aux_sym_link_label_repeat1 = 168, - aux_sym_link_destination_repeat1 = 169, - aux_sym_link_destination_repeat2 = 170, - aux_sym_link_title_repeat1 = 171, - aux_sym_link_title_repeat2 = 172, - aux_sym_link_title_repeat3 = 173, - aux_sym__section1_repeat1 = 174, - aux_sym__section2_repeat1 = 175, - aux_sym__section3_repeat1 = 176, - aux_sym__section4_repeat1 = 177, - aux_sym__section5_repeat1 = 178, - aux_sym_indented_code_block_repeat1 = 179, - aux_sym__indented_chunk_repeat1 = 180, - aux_sym_code_fence_content_repeat1 = 181, - aux_sym_info_string_repeat1 = 182, - aux_sym_info_string_repeat2 = 183, - aux_sym_language_repeat1 = 184, - aux_sym__html_block_1_repeat1 = 185, - aux_sym__html_block_2_repeat1 = 186, - aux_sym__html_block_3_repeat1 = 187, - aux_sym__html_block_4_repeat1 = 188, - aux_sym__html_block_5_repeat1 = 189, - aux_sym__html_block_6_repeat1 = 190, - aux_sym_paragraph_repeat1 = 191, - aux_sym_block_quote_repeat1 = 192, - aux_sym__list_plus_repeat1 = 193, - aux_sym__list_minus_repeat1 = 194, - aux_sym__list_star_repeat1 = 195, - aux_sym__list_dot_repeat1 = 196, - aux_sym__list_parenthesis_repeat1 = 197, - aux_sym__line_repeat1 = 198, - aux_sym_pipe_table_repeat1 = 199, - aux_sym_pipe_table_delimiter_row_repeat1 = 200, - aux_sym_pipe_table_delimiter_cell_repeat1 = 201, - aux_sym_pipe_table_row_repeat1 = 202, - aux_sym_pipe_table_cell_repeat1 = 203, - alias_sym_inline = 204, - alias_sym_pipe_table_align_left = 205, - alias_sym_pipe_table_align_right = 206, - alias_sym_pipe_table_header = 207, + anon_sym_DASH_DASH_GT = 36, + anon_sym_QMARK_GT = 37, + anon_sym_RBRACK_RBRACK_GT = 38, + aux_sym__word_token1 = 39, + aux_sym__word_token2 = 40, + aux_sym__word_token3 = 41, + sym__whitespace = 42, + sym__line_ending = 43, + sym__soft_line_ending = 44, + sym__block_close = 45, + sym_block_continuation = 46, + sym__block_quote_start = 47, + sym__indented_chunk_start = 48, + sym_atx_h1_marker = 49, + sym_atx_h2_marker = 50, + sym_atx_h3_marker = 51, + sym_atx_h4_marker = 52, + sym_atx_h5_marker = 53, + sym_atx_h6_marker = 54, + sym_setext_h1_underline = 55, + sym_setext_h2_underline = 56, + sym__thematic_break = 57, + sym__list_marker_minus = 58, + sym__list_marker_plus = 59, + sym__list_marker_star = 60, + sym__list_marker_parenthesis = 61, + sym__list_marker_dot = 62, + sym__list_marker_minus_dont_interrupt = 63, + sym__list_marker_plus_dont_interrupt = 64, + sym__list_marker_star_dont_interrupt = 65, + sym__list_marker_parenthesis_dont_interrupt = 66, + sym__list_marker_dot_dont_interrupt = 67, + sym__fenced_code_block_start_backtick = 68, + sym__fenced_code_block_start_tilde = 69, + sym__blank_line_start = 70, + sym__fenced_code_block_end_backtick = 71, + sym__fenced_code_block_end_tilde = 72, + sym__html_block_1_start = 73, + sym__html_block_1_end = 74, + sym__html_block_2_start = 75, + sym__html_block_3_start = 76, + sym__html_block_4_start = 77, + sym__html_block_5_start = 78, + sym__html_block_6_start = 79, + sym__html_block_7_start = 80, + sym__close_block = 81, + sym__no_indented_chunk = 82, + sym__error = 83, + sym__trigger_error = 84, + sym__eof = 85, + sym_minus_metadata = 86, + sym_plus_metadata = 87, + sym__pipe_table_start = 88, + sym__pipe_table_line_ending = 89, + sym_document = 90, + sym_backslash_escape = 91, + sym_link_label = 92, + sym_link_destination = 93, + sym__link_destination_parenthesis = 94, + sym__text_no_angle = 95, + sym_link_title = 96, + sym__last_token_punctuation = 97, + sym__block = 98, + sym__block_not_section = 99, + sym_section = 100, + sym__section1 = 101, + sym__section2 = 102, + sym__section3 = 103, + sym__section4 = 104, + sym__section5 = 105, + sym__section6 = 106, + sym_thematic_break = 107, + sym__atx_heading1 = 108, + sym__atx_heading2 = 109, + sym__atx_heading3 = 110, + sym__atx_heading4 = 111, + sym__atx_heading5 = 112, + sym__atx_heading6 = 113, + sym__atx_heading_content = 114, + sym__setext_heading1 = 115, + sym__setext_heading2 = 116, + sym_indented_code_block = 117, + sym__indented_chunk = 118, + sym_fenced_code_block = 119, + sym_code_fence_content = 120, + sym_info_string = 121, + sym_language = 122, + sym_html_block = 123, + sym__html_block_1 = 124, + sym__html_block_2 = 125, + sym__html_block_3 = 126, + sym__html_block_4 = 127, + sym__html_block_5 = 128, + sym__html_block_6 = 129, + sym__html_block_7 = 130, + sym_link_reference_definition = 131, + sym__text_inline_no_link = 132, + sym_paragraph = 133, + sym__blank_line = 134, + sym_block_quote = 135, + sym_list = 136, + sym__list_plus = 137, + sym__list_minus = 138, + sym__list_star = 139, + sym__list_dot = 140, + sym__list_parenthesis = 141, + sym_list_marker_plus = 142, + sym_list_marker_minus = 143, + sym_list_marker_star = 144, + sym_list_marker_dot = 145, + sym_list_marker_parenthesis = 146, + sym__list_item_plus = 147, + sym__list_item_minus = 148, + sym__list_item_star = 149, + sym__list_item_dot = 150, + sym__list_item_parenthesis = 151, + sym__list_item_content = 152, + sym__newline = 153, + sym__soft_line_break = 154, + sym__line = 155, + sym__word = 156, + sym_task_list_marker_checked = 157, + sym_task_list_marker_unchecked = 158, + sym_pipe_table = 159, + sym__pipe_table_newline = 160, + sym_pipe_table_delimiter_row = 161, + sym_pipe_table_delimiter_cell = 162, + sym_pipe_table_row = 163, + sym_pipe_table_cell = 164, + aux_sym_document_repeat1 = 165, + aux_sym_document_repeat2 = 166, + aux_sym_link_label_repeat1 = 167, + aux_sym_link_destination_repeat1 = 168, + aux_sym_link_destination_repeat2 = 169, + aux_sym_link_title_repeat1 = 170, + aux_sym_link_title_repeat2 = 171, + aux_sym_link_title_repeat3 = 172, + aux_sym__section1_repeat1 = 173, + aux_sym__section2_repeat1 = 174, + aux_sym__section3_repeat1 = 175, + aux_sym__section4_repeat1 = 176, + aux_sym__section5_repeat1 = 177, + aux_sym_indented_code_block_repeat1 = 178, + aux_sym__indented_chunk_repeat1 = 179, + aux_sym_code_fence_content_repeat1 = 180, + aux_sym_info_string_repeat1 = 181, + aux_sym_info_string_repeat2 = 182, + aux_sym_language_repeat1 = 183, + aux_sym__html_block_1_repeat1 = 184, + aux_sym__html_block_2_repeat1 = 185, + aux_sym__html_block_3_repeat1 = 186, + aux_sym__html_block_4_repeat1 = 187, + aux_sym__html_block_5_repeat1 = 188, + aux_sym__html_block_6_repeat1 = 189, + aux_sym_paragraph_repeat1 = 190, + aux_sym_block_quote_repeat1 = 191, + aux_sym__list_plus_repeat1 = 192, + aux_sym__list_minus_repeat1 = 193, + aux_sym__list_star_repeat1 = 194, + aux_sym__list_dot_repeat1 = 195, + aux_sym__list_parenthesis_repeat1 = 196, + aux_sym__line_repeat1 = 197, + aux_sym_pipe_table_repeat1 = 198, + aux_sym_pipe_table_delimiter_row_repeat1 = 199, + aux_sym_pipe_table_delimiter_cell_repeat1 = 200, + aux_sym_pipe_table_row_repeat1 = 201, + aux_sym_pipe_table_cell_repeat1 = 202, + alias_sym_inline = 203, + alias_sym_pipe_table_align_left = 204, + alias_sym_pipe_table_align_right = 205, + alias_sym_pipe_table_header = 206, }; static const char * const ts_symbol_names[] = { @@ -270,7 +269,6 @@ static const char * const ts_symbol_names[] = { [anon_sym_TILDE] = "~", [anon_sym_LPAREN] = "(", [anon_sym_RPAREN] = ")", - [sym__newline_token] = "_newline_token", [anon_sym_DASH_DASH_GT] = "-->", [anon_sym_QMARK_GT] = "\?>", [anon_sym_RBRACK_RBRACK_GT] = "]]>", @@ -481,7 +479,6 @@ static const TSSymbol ts_symbol_map[] = { [anon_sym_TILDE] = anon_sym_TILDE, [anon_sym_LPAREN] = anon_sym_LPAREN, [anon_sym_RPAREN] = anon_sym_RPAREN, - [sym__newline_token] = sym__newline_token, [anon_sym_DASH_DASH_GT] = anon_sym_DASH_DASH_GT, [anon_sym_QMARK_GT] = anon_sym_QMARK_GT, [anon_sym_RBRACK_RBRACK_GT] = anon_sym_RBRACK_RBRACK_GT, @@ -800,10 +797,6 @@ static const TSSymbolMetadata ts_symbol_metadata[] = { .visible = true, .named = false, }, - [sym__newline_token] = { - .visible = false, - .named = true, - }, [anon_sym_DASH_DASH_GT] = { .visible = true, .named = false, @@ -2519,8 +2512,6 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { case 0: if (eof) ADVANCE(2031); ADVANCE_MAP( - '\n', 2072, - '\r', 2073, '!', 2040, '"', 2041, '#', 2042, @@ -2553,10 +2544,13 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { '|', 2067, '}', 2068, '~', 2069, - '\t', 2080, - ' ', 2080, + '\t', 2078, + ' ', 2078, ); - if (lookahead != 0) ADVANCE(2077); + if (lookahead != 0 && + lookahead != '\t' && + lookahead != '\n' && + lookahead != '\r') ADVANCE(2075); END_STATE(); case 1: ADVANCE_MAP( @@ -2592,13 +2586,13 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { '|', 2067, '}', 2068, '~', 2069, - '\t', 2080, - ' ', 2080, + '\t', 2078, + ' ', 2078, ); if (lookahead != 0 && lookahead != '\t' && lookahead != '\n' && - lookahead != '\r') ADVANCE(2077); + lookahead != '\r') ADVANCE(2075); END_STATE(); case 2: ADVANCE_MAP( @@ -2634,13 +2628,13 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { '|', 2067, '}', 2068, '~', 2069, - '\t', 2080, - ' ', 2080, + '\t', 2078, + ' ', 2078, ); if (lookahead != 0 && lookahead != '\t' && lookahead != '\n' && - lookahead != '\r') ADVANCE(2077); + lookahead != '\r') ADVANCE(2075); END_STATE(); case 3: ADVANCE_MAP( @@ -2676,13 +2670,13 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { '|', 2067, '}', 2068, '~', 2069, - '\t', 2080, - ' ', 2080, + '\t', 2078, + ' ', 2078, ); if (lookahead != 0 && lookahead != '\t' && lookahead != '\n' && - lookahead != '\r') ADVANCE(2077); + lookahead != '\r') ADVANCE(2075); END_STATE(); case 4: ADVANCE_MAP( @@ -2718,13 +2712,13 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { '|', 2067, '}', 2068, '~', 2069, - '\t', 2080, - ' ', 2080, + '\t', 2078, + ' ', 2078, ); if (lookahead != 0 && lookahead != '\t' && lookahead != '\n' && - lookahead != '\r') ADVANCE(2077); + lookahead != '\r') ADVANCE(2075); END_STATE(); case 5: ADVANCE_MAP( @@ -2760,13 +2754,13 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { '|', 2067, '}', 2068, '~', 2069, - '\t', 2080, - ' ', 2080, + '\t', 2078, + ' ', 2078, ); if (lookahead != 0 && lookahead != '\t' && lookahead != '\n' && - lookahead != '\r') ADVANCE(2077); + lookahead != '\r') ADVANCE(2075); END_STATE(); case 6: if (lookahead == '1') ADVANCE(2010); @@ -4110,10 +4104,10 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { ('a' <= lookahead && lookahead <= 'f')) ADVANCE(245); END_STATE(); case 247: - if (lookahead == '>') ADVANCE(2074); + if (lookahead == '>') ADVANCE(2072); END_STATE(); case 248: - if (lookahead == '>') ADVANCE(2076); + if (lookahead == '>') ADVANCE(2074); END_STATE(); case 249: ADVANCE_MAP( @@ -5031,10 +5025,10 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { if (lookahead == 'W') ADVANCE(1127); END_STATE(); case 380: - if (lookahead == ']') ADVANCE(2079); + if (lookahead == ']') ADVANCE(2077); END_STATE(); case 381: - if (lookahead == ']') ADVANCE(2078); + if (lookahead == ']') ADVANCE(2076); END_STATE(); case 382: ADVANCE_MAP( @@ -10830,13 +10824,13 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { '|', 2067, '}', 2068, '~', 2069, - '\t', 2080, - ' ', 2080, + '\t', 2078, + ' ', 2078, ); if (lookahead != 0 && lookahead != '\t' && lookahead != '\n' && - lookahead != '\r') ADVANCE(2077); + lookahead != '\r') ADVANCE(2075); END_STATE(); case 2031: ACCEPT_TOKEN(ts_builtin_sym_end); @@ -10985,7 +10979,7 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { END_STATE(); case 2059: ACCEPT_TOKEN(anon_sym_QMARK); - if (lookahead == '>') ADVANCE(2075); + if (lookahead == '>') ADVANCE(2073); END_STATE(); case 2060: ACCEPT_TOKEN(anon_sym_AT); @@ -11028,22 +11022,15 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { ACCEPT_TOKEN(anon_sym_RPAREN); END_STATE(); case 2072: - ACCEPT_TOKEN(sym__newline_token); - END_STATE(); - case 2073: - ACCEPT_TOKEN(sym__newline_token); - if (lookahead == '\n') ADVANCE(2072); - END_STATE(); - case 2074: ACCEPT_TOKEN(anon_sym_DASH_DASH_GT); END_STATE(); - case 2075: + case 2073: ACCEPT_TOKEN(anon_sym_QMARK_GT); END_STATE(); - case 2076: + case 2074: ACCEPT_TOKEN(anon_sym_RBRACK_RBRACK_GT); END_STATE(); - case 2077: + case 2075: ACCEPT_TOKEN(aux_sym__word_token1); if (lookahead != 0 && lookahead != '\t' && @@ -11052,18 +11039,18 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { (lookahead < ' ' || '/' < lookahead) && (lookahead < ':' || '@' < lookahead) && (lookahead < '[' || '`' < lookahead) && - (lookahead < '{' || '~' < lookahead)) ADVANCE(2077); + (lookahead < '{' || '~' < lookahead)) ADVANCE(2075); END_STATE(); - case 2078: + case 2076: ACCEPT_TOKEN(aux_sym__word_token2); END_STATE(); - case 2079: + case 2077: ACCEPT_TOKEN(aux_sym__word_token3); END_STATE(); - case 2080: + case 2078: ACCEPT_TOKEN(sym__whitespace); if (lookahead == '\t' || - lookahead == ' ') ADVANCE(2080); + lookahead == ' ') ADVANCE(2078); END_STATE(); default: return false; @@ -12036,7 +12023,6 @@ static const uint16_t ts_parse_table[LARGE_STATE_COUNT][SYMBOL_COUNT] = { [anon_sym_TILDE] = ACTIONS(1), [anon_sym_LPAREN] = ACTIONS(1), [anon_sym_RPAREN] = ACTIONS(1), - [sym__newline_token] = ACTIONS(1), [anon_sym_DASH_DASH_GT] = ACTIONS(1), [anon_sym_QMARK_GT] = ACTIONS(1), [anon_sym_RBRACK_RBRACK_GT] = ACTIONS(1), diff --git a/tree-sitter-markdown/src/tree_sitter/alloc.h b/tree-sitter-markdown/src/tree_sitter/alloc.h index 1f4466d..1abdd12 100644 --- a/tree-sitter-markdown/src/tree_sitter/alloc.h +++ b/tree-sitter-markdown/src/tree_sitter/alloc.h @@ -12,10 +12,10 @@ extern "C" { // Allow clients to override allocation functions #ifdef TREE_SITTER_REUSE_ALLOCATOR -extern void *(*ts_current_malloc)(size_t); -extern void *(*ts_current_calloc)(size_t, size_t); -extern void *(*ts_current_realloc)(void *, size_t); -extern void (*ts_current_free)(void *); +extern void *(*ts_current_malloc)(size_t size); +extern void *(*ts_current_calloc)(size_t count, size_t size); +extern void *(*ts_current_realloc)(void *ptr, size_t size); +extern void (*ts_current_free)(void *ptr); #ifndef ts_malloc #define ts_malloc ts_current_malloc diff --git a/tree-sitter.json b/tree-sitter.json new file mode 100644 index 0000000..27d389b --- /dev/null +++ b/tree-sitter.json @@ -0,0 +1,42 @@ +{ + "metadata": { + "version": "0.4.0", + "license": "MIT", + "authors": [ + { + "name": "MDeiml" + } + ], + "description": "Markdown grammar for tree-sitter", + "links": { + "repository": "https://github.com/tree-sitter-grammars/tree-sitter-markdown" + } + }, + "grammars": [ + { + "name": "markdown", + "scope": "text.markdown", + "path": "tree-sitter-markdown", + "injection-regex": "^(markdown|md)$", + "file-types": [ + "md" + ], + "highlights": "queries/highlights.scm", + "injections": "queries/injections.scm", + "external-files": [ + "common/common.js" + ] + }, + { + "name": "markdown_inline", + "camelcase": "MarkdownInline", + "scope": "text.markdown_inline", + "path": "tree-sitter-markdown-inline", + "highlights": "queries/highlights.scm", + "injections": "queries/injections.scm", + "external-files": [ + "common/common.js" + ] + } + ] +}