From e7f29417bf7d8e6e93caee0a886191ef000f4e57 Mon Sep 17 00:00:00 2001 From: "http://jneen.net/" Date: Sat, 18 Jul 2020 02:05:40 -0400 Subject: [PATCH 1/5] implement a json5 lexer --- lib/rouge/demos/json5 | 12 +++++++ lib/rouge/lexers/json5.rb | 75 +++++++++++++++++++++++++++++++++++++++ spec/lexers/json5_spec.rb | 18 ++++++++++ spec/visual/samples/json5 | 44 +++++++++++++++++++++++ 4 files changed, 149 insertions(+) create mode 100644 lib/rouge/demos/json5 create mode 100644 lib/rouge/lexers/json5.rb create mode 100644 spec/lexers/json5_spec.rb create mode 100644 spec/visual/samples/json5 diff --git a/lib/rouge/demos/json5 b/lib/rouge/demos/json5 new file mode 100644 index 0000000000..0e72c64622 --- /dev/null +++ b/lib/rouge/demos/json5 @@ -0,0 +1,12 @@ +{ + // comments + unquoted: 'and you can quote me on that', + singleQuotes: 'I can use "double quotes" here', + lineBreaks: "Look, Mom! \ +No \\n's!", + hexadecimal: 0xdecaf, + leadingDecimalPoint: .8675309, andTrailing: 8675309., + positiveSign: +1, + trailingComma: 'in objects', andIn: ['arrays',], + "backwardsCompatible": "with JSON", +} diff --git a/lib/rouge/lexers/json5.rb b/lib/rouge/lexers/json5.rb new file mode 100644 index 0000000000..d9b0452d89 --- /dev/null +++ b/lib/rouge/lexers/json5.rb @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- # +# frozen_string_literal: true + +module Rouge + module Lexers + load_lexer 'javascript.rb' + + class JSON5 < JSON + title 'JSON 5' + tag 'json5' + filenames '*.json5' + mimetypes 'application/json5', 'application/x-json5' + + append :whitespace do + rule %r://.*$:, Comment + + # comments are non-nesting, so a single regex will do + rule %r:/[*].*?[*]/:m, Comment + end + + prepend :name do + rule Javascript.id_regex, Name::Label + rule %r/".*?"/, Name::Label + end + + # comments can appear between keys and :, so we have to + # manage our states a little more carefully + append :object do + rule %r/:/ do + token Punctuation + goto :object_value + end + end + + state :object_value do + mixin :value + rule %r/,/ do + token Punctuation + goto :object + end + + rule %r/}/, Punctuation, :pop! + end + + append :value do + rule %r/'/, Str::Single, :sstring + end + + state :sstring do + rule %r/[^\\']+/, Str::Single + rule %r/\\./m, Str::Escape + rule %r/'/, Str::Single, :pop! + end + + # can escape newlines + append :string do + rule %r/\\./m, Str::Escape + end + + # override: numbers are very different in json5 + state :constants do + rule %r/\b(?:true|false|null)\b/, Keyword::Constant + rule %r/[+-]?\b(?:Infinity|NaN)\b/, Keyword::Constant + rule %r/[+-]?0x\h+/i, Num::Hex + + rule %r/[+-]?\d*\.\d*(?:e[+-]?\d+)?/i, Num::Float + rule %r/[+-]?\d+e[+-]?\d+/, Num::Integer + rule %r/[+-]?(?:0|[1-9]\d*)(?:e[+-]?\d+)?/i, Num::Integer + + rule %r/[+-]?[.]\d+/, Num::Float + rule %r/[+-]?\d+[.]/, Num::Float + end + end + end +end diff --git a/spec/lexers/json5_spec.rb b/spec/lexers/json5_spec.rb new file mode 100644 index 0000000000..44feb7943e --- /dev/null +++ b/spec/lexers/json5_spec.rb @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- # +# frozen_string_literal: true + +describe Rouge::Lexers::JSON5 do + let(:subject) { Rouge::Lexers::JSON5.new } + + describe 'guessing' do + include Support::Guessing + + it 'guesses by filename' do + assert_guess :filename => 'foo.json5' + end + + it 'guesses by mimetype' do + assert_guess :mimetype => 'application/json5' + end + end +end diff --git a/spec/visual/samples/json5 b/spec/visual/samples/json5 new file mode 100644 index 0000000000..90c725d32e --- /dev/null +++ b/spec/visual/samples/json5 @@ -0,0 +1,44 @@ +{ + // comments + unquoted: 'and you can quote me on that', + singleQuotes: 'I can use "double quotes" here', + lineBreaks: "Look, Mom! \ +No \\n's!", + hexadecimal: 0xdecaf, + leadingDecimalPoint: .8675309, andTrailing: 8675309., + positiveSign: +1, + trailingComma: 'in objects', andIn: ['arrays',], + "backwardsCompatible": "with JSON", + + + a: 1e5, + b: 1.e-5, + c: .1e5, + + /** + * multiline + * comments + */ + + d + : 3, + + "e" + // comment + : 4, + + foo: "bar", + "bar": "foo", + + f //comment + : 5, + + constants: [ + NaN, Infinity, true, false, null, + { NaN: NaN, + Infinity: Infinity, + true: true, + false: false, + null: null }, + ] +} From 94ca37b077ab93592df59461667aaedd29140df9 Mon Sep 17 00:00:00 2001 From: "http://jneen.net/" Date: Sat, 18 Jul 2020 02:07:00 -0400 Subject: [PATCH 2/5] add a desc --- lib/rouge/lexers/json5.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/rouge/lexers/json5.rb b/lib/rouge/lexers/json5.rb index d9b0452d89..f383b1682c 100644 --- a/lib/rouge/lexers/json5.rb +++ b/lib/rouge/lexers/json5.rb @@ -11,6 +11,8 @@ class JSON5 < JSON filenames '*.json5' mimetypes 'application/json5', 'application/x-json5' + desc 'JSON 5 extension for JSON (json5.org)' + append :whitespace do rule %r://.*$:, Comment From 89624a3fafae2b7ff7f326a45b22bba00425c56a Mon Sep 17 00:00:00 2001 From: Michael Camilleri Date: Tue, 8 Sep 2020 12:48:11 +0900 Subject: [PATCH 3/5] Remove unnecessary rules --- lib/rouge/lexers/json5.rb | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/rouge/lexers/json5.rb b/lib/rouge/lexers/json5.rb index f383b1682c..0f78818938 100644 --- a/lib/rouge/lexers/json5.rb +++ b/lib/rouge/lexers/json5.rb @@ -68,9 +68,6 @@ class JSON5 < JSON rule %r/[+-]?\d*\.\d*(?:e[+-]?\d+)?/i, Num::Float rule %r/[+-]?\d+e[+-]?\d+/, Num::Integer rule %r/[+-]?(?:0|[1-9]\d*)(?:e[+-]?\d+)?/i, Num::Integer - - rule %r/[+-]?[.]\d+/, Num::Float - rule %r/[+-]?\d+[.]/, Num::Float end end end From 1aeddb50d91b9e061cb4e77b5db8d4af838a0338 Mon Sep 17 00:00:00 2001 From: Tan Le Date: Mon, 11 Nov 2024 11:31:13 +1000 Subject: [PATCH 4/5] Mirror pattern from Pygments JSON5 --- lib/rouge/lexers/json5.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rouge/lexers/json5.rb b/lib/rouge/lexers/json5.rb index 0f78818938..99185cf70c 100644 --- a/lib/rouge/lexers/json5.rb +++ b/lib/rouge/lexers/json5.rb @@ -65,7 +65,7 @@ class JSON5 < JSON rule %r/[+-]?\b(?:Infinity|NaN)\b/, Keyword::Constant rule %r/[+-]?0x\h+/i, Num::Hex - rule %r/[+-]?\d*\.\d*(?:e[+-]?\d+)?/i, Num::Float + rule %r/[+-.]?[0-9]+[.]?[0-9]?([eE][-]?[0-9]+)?/i, Num::Float rule %r/[+-]?\d+e[+-]?\d+/, Num::Integer rule %r/[+-]?(?:0|[1-9]\d*)(?:e[+-]?\d+)?/i, Num::Integer end From 04cdbeb4e8ebcc81508d71d04256543a81894901 Mon Sep 17 00:00:00 2001 From: Tan Le Date: Mon, 11 Nov 2024 11:37:56 +1000 Subject: [PATCH 5/5] Document JSON5 support --- docs/Languages.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/Languages.md b/docs/Languages.md index 18864ebe0a..4818b54e62 100644 --- a/docs/Languages.md +++ b/docs/Languages.md @@ -103,6 +103,7 @@ - Jinja (`jinja`) - JSL (`jsl`) - JSON (`json`) +- JSON5 (`json5`) - Json-doc (`json-doc`) - Jsonnet (`jsonnet`) - Jsp (`jsp`)