Translate vscode-textmate from TypeScript to Java to support

tokenizeLine2. See #38
eclipse-tm4e · May 10, 2017 · b28d76f · b28d76f
1 parent 81fcb8d
commit b28d76f
Show file tree

Hide file tree

Showing 335 changed files with 1,185,063 additions and 394 deletions.
diff --git a/....tm4e.core.tests/src/test/java/org/eclipse/tm4e/core/tests/grammar/VSCodeGrammarTest.java b/....tm4e.core.tests/src/test/java/org/eclipse/tm4e/core/tests/grammar/VSCodeGrammarTest.java
@@ -0,0 +1,116 @@
+/**
+ *  Copyright (c) 2015-2017 Angelo ZERR.
+ *  All rights reserved. This program and the accompanying materials
+ *  are made available under the terms of the Eclipse Public License v1.0
+ *  which accompanies this distribution, and is available at
+ *  http://www.eclipse.org/legal/epl-v10.html
+ *
+ *  Contributors:
+ *  Angelo Zerr <[email protected]> - initial API and implementation
+ */
+package org.eclipse.tm4e.core.tests.grammar;
+
+import org.eclipse.tm4e.core.internal.grammar.StackElementMetadata;
+import org.eclipse.tm4e.core.internal.grammar.StandardTokenType;
+import org.eclipse.tm4e.core.theme.FontStyle;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * {@link StackElementMetadata} tests same than vscode-textmate.
+ * 
+ * @see https://github.com/Microsoft/vscode-textmate/blob/master/src/tests/grammar.test.ts
+ *
+ */
+public class VSCodeGrammarTest {
+
+	@Test
+	public void works() {
+		int value = StackElementMetadata.set(0, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101,
+				102);
+		assertEquals(value, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101, 102);
+	}
+
+	@Test
+	public void canOverwriteLanguageId() {
+		int value = StackElementMetadata.set(0, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101,
+				102);
+		assertEquals(value, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101, 102);
+
+		value = StackElementMetadata.set(value, 2, StandardTokenType.Other, FontStyle.NotSet, 0, 0);
+		assertEquals(value, 2, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101, 102);
+	}
+
+	@Test
+	public void canOverwriteTokenType() {
+		int value = StackElementMetadata.set(0, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101,
+				102);
+		assertEquals(value, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101, 102);
+
+		value = StackElementMetadata.set(value, 0, StandardTokenType.Comment, FontStyle.NotSet, 0, 0);
+		assertEquals(value, 1, StandardTokenType.Comment, FontStyle.Underline | FontStyle.Bold, 101, 102);
+	}
+
+	@Test
+	public void canOverwriteFontStyle() {
+		int value = StackElementMetadata.set(0, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101,
+				102);
+		assertEquals(value, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101, 102);
+
+		value = StackElementMetadata.set(value, 0, StandardTokenType.Other, FontStyle.None, 0, 0);
+		assertEquals(value, 1, StandardTokenType.RegEx, FontStyle.None, 101, 102);
+	}
+
+	@Test
+	public void canOverwriteForeground() {
+		int value = StackElementMetadata.set(0, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101,
+				102);
+		assertEquals(value, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101, 102);
+
+		value = StackElementMetadata.set(value, 0, StandardTokenType.Other, FontStyle.NotSet, 5, 0);
+		assertEquals(value, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 5, 102);
+	}
+
+	@Test
+	public void canOverwriteBackground() {
+		int value = StackElementMetadata.set(0, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101,
+				102);
+		assertEquals(value, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101, 102);
+
+		value = StackElementMetadata.set(value, 0, StandardTokenType.Other, FontStyle.NotSet, 0, 7);
+		assertEquals(value, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101, 7);
+	}
+
+	@Test
+	public void canWorkAtMaxValues() {
+		int maxLangId = 255;
+		int maxTokenType = StandardTokenType.Comment | StandardTokenType.Other | StandardTokenType.RegEx
+				| StandardTokenType.String;
+		int maxFontStyle = FontStyle.Bold | FontStyle.Italic | FontStyle.Underline;
+		int maxForeground = 511;
+		int maxBackground = 511;
+
+		int value = StackElementMetadata.set(0, maxLangId, maxTokenType, maxFontStyle, maxForeground, maxBackground);
+		assertEquals(value, maxLangId, maxTokenType, maxFontStyle, maxForeground, maxBackground);
+	}
+
+	private static void assertEquals(int metadata, int languageId, int tokenType, int fontStyle, int foreground, int background) {
+		String actual = "{\n" + 
+			"languageId: " + StackElementMetadata.getLanguageId(metadata) + ",\n" +
+			"tokenType: " + StackElementMetadata.getTokenType(metadata) + ",\n" +
+			"fontStyle: " + StackElementMetadata.getFontStyle(metadata) + ",\n" +
+			"foreground: " + StackElementMetadata.getForeground(metadata) + ",\n" +
+			"background: " + StackElementMetadata.getBackground(metadata) + ",\n" +			
+		"}";
+
+		String expected = "{\n" + 
+				"languageId: " + languageId + ",\n" +
+				"tokenType: " + tokenType + ",\n" +
+				"fontStyle: " + fontStyle + ",\n" +
+				"foreground: " + foreground + ",\n" +
+				"background: " + background + ",\n" +			
+			"}";
+
+		Assert.assertEquals("equals for " + StackElementMetadata.toBinaryStr(metadata), expected, actual);
+	}
+}
diff --git a/...core/grammar/test/VSCodeTextMateTest.java → ...ore/tests/grammar/VSCodeTextMateTest.java b/...core/grammar/test/VSCodeTextMateTest.java → ...ore/tests/grammar/VSCodeTextMateTest.java
@@ -8,7 +8,7 @@
  *  Contributors:
  *  Angelo Zerr <[email protected]> - initial API and implementation
  */
-package org.eclipse.tm4e.core.grammar.test;
+package org.eclipse.tm4e.core.tests.grammar;
 
 import java.io.File;
 import java.io.FileReader;
@@ -17,6 +17,8 @@
 import java.util.Arrays;
 import java.util.List;
 
+import org.eclipse.tm4e.core.tests.grammar.internal.MatcherTestImpl;
+import org.eclipse.tm4e.core.tests.grammar.internal.RawTestImpl;
 import org.junit.runner.RunWith;
 import org.junit.runners.AllTests;
 
@@ -25,6 +27,13 @@
 
 import junit.framework.TestSuite;
 
+/**
+ * VSCode TextMate grammar tests which uses same vscode-textmate tests located
+ * at src\test\resources\test-cases
+ * 
+ * @see https://github.com/Microsoft/vscode-textmate/blob/master/src/tests/tests.ts
+ *
+ */
 @RunWith(AllTests.class)
 public class VSCodeTextMateTest {
 

diff --git a/...4e/core/grammar/test/MatcherTestImpl.java → ...sts/grammar/internal/MatcherTestImpl.java b/...4e/core/grammar/test/MatcherTestImpl.java → ...sts/grammar/internal/MatcherTestImpl.java
@@ -8,7 +8,7 @@
  *  Contributors:
  *  Angelo Zerr <[email protected]> - initial API and implementation
  */
-package org.eclipse.tm4e.core.grammar.test;
+package org.eclipse.tm4e.core.tests.grammar.internal;
 
 import java.util.Collection;
 import java.util.List;

diff --git a/...e/tm4e/core/grammar/test/RawTestImpl.java → ...e/tests/grammar/internal/RawTestImpl.java b/...e/tm4e/core/grammar/test/RawTestImpl.java → ...e/tests/grammar/internal/RawTestImpl.java
@@ -8,7 +8,7 @@
  *  Contributors:
  *  Angelo Zerr <[email protected]> - initial API and implementation
  */
-package org.eclipse.tm4e.core.grammar.test;
+package org.eclipse.tm4e.core.tests.grammar.internal;
 
 import java.io.File;
 import java.io.IOException;
@@ -23,7 +23,7 @@
 import org.eclipse.tm4e.core.grammar.ITokenizeLineResult;
 import org.eclipse.tm4e.core.grammar.StackElement;
 import org.eclipse.tm4e.core.logger.SystemLogger;
-import org.eclipse.tm4e.core.registry.IGrammarLocator;
+import org.eclipse.tm4e.core.registry.IRegistryOptions;
 import org.eclipse.tm4e.core.registry.Registry;
 import org.junit.Assert;
 import org.junit.runner.Describable;
@@ -84,7 +84,7 @@ public int countTestCases() {
 	}
 
 	private static void executeTest(RawTestImpl test, File testLocation) throws Exception {
-		IGrammarLocator locator = new IGrammarLocator() {
+		IRegistryOptions locator = new IRegistryOptions() {
 
 			@Override
 			public String getFilePath(String scopeName) {

diff --git a/...e/tm4e/core/grammar/test/RawTestLine.java → ...e/tests/grammar/internal/RawTestLine.java b/...e/tm4e/core/grammar/test/RawTestLine.java → ...e/tests/grammar/internal/RawTestLine.java
@@ -8,7 +8,7 @@
  *  Contributors:
  *  Angelo Zerr <[email protected]> - initial API and implementation
  */
-package org.eclipse.tm4e.core.grammar.test;
+package org.eclipse.tm4e.core.tests.grammar.internal;
 
 import java.util.List;
 

diff --git a/...ipse/tm4e/core/grammar/test/RawToken.java → ...core/tests/grammar/internal/RawToken.java b/...ipse/tm4e/core/grammar/test/RawToken.java → ...core/tests/grammar/internal/RawToken.java
@@ -8,7 +8,7 @@
  *  Contributors:
  *  Angelo Zerr <[email protected]> - initial API and implementation
  */
-package org.eclipse.tm4e.core.grammar.test;
+package org.eclipse.tm4e.core.tests.grammar.internal;
 
 import java.util.List;
 

diff --git a/...ipse.tm4e.core.tests/src/test/java/org/eclipse/tm4e/core/tests/theme/VSCodeThemeTest.java b/...ipse.tm4e.core.tests/src/test/java/org/eclipse/tm4e/core/tests/theme/VSCodeThemeTest.java
@@ -0,0 +1,54 @@
+package org.eclipse.tm4e.core.tests.theme;
+
+import java.io.ByteArrayInputStream;
+import java.util.Arrays;
+import java.util.List;
+
+import org.eclipse.tm4e.core.internal.theme.reader.ThemeReader;
+import org.eclipse.tm4e.core.theme.ColorMap;
+import org.eclipse.tm4e.core.theme.FontStyle;
+import org.eclipse.tm4e.core.theme.Theme;
+import org.eclipse.tm4e.core.theme.ThemeTrieElementRule;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * 
+ * @see https://github.com/Microsoft/vscode-textmate/blob/master/src/tests/themes.test.ts
+ *
+ */
+public class VSCodeThemeTest {
+
+	@Test
+	public void givesHigherPriorityToDeeperMatches() throws Exception {
+		Theme theme = loadTheme("{" + 
+			"\"settings\": ["+
+						"{ \"settings\": { \"foreground\": \"#100000\", \"background\": \"#200000\" } },"+
+						"{ \"scope\": \"punctuation.definition.string.begin.html\", \"settings\": { \"foreground\": \"#300000\" } },"+
+						"{ \"scope\": \"meta.tag punctuation.definition.string\", \"settings\": { \"foreground\": \"#400000\" } }"+
+						// { scope: 'a', settings: { foreground: '#500000' } },
+					"]"+
+				"}");
+
+
+		ColorMap colorMap = new ColorMap();
+		int _NOT_SET = 0;
+		int _A = colorMap.getId("#100000");
+		int _B = colorMap.getId("#200000");
+		int _C = colorMap.getId("#400000");
+		int _D = colorMap.getId("#300000");
+
+		List<ThemeTrieElementRule> actual = theme.match("punctuation.definition.string.begin.html");
+
+		/*Assert.assertEquals(actual, Arrays.asList(
+			new ThemeTrieElementRule(5, null, FontStyle.NotSet, _D, _NOT_SET),
+			new ThemeTrieElementRule(3, Arrays.asList("meta.tag"), FontStyle.NotSet, _C, _NOT_SET)
+		));*/
+
+
+	}
+
+	private Theme loadTheme(String theme) throws Exception {
+		return Theme.createFromRawTheme(ThemeReader.JSON_PARSER.parse(new ByteArrayInputStream(theme.getBytes())));
+	}
+}
diff --git a/org.eclipse.tm4e.core.tests/src/test/resources/test-cases/suite1/tests.json b/org.eclipse.tm4e.core.tests/src/test/resources/test-cases/suite1/tests.json
@@ -591,7 +591,8 @@
 						"value": "html",
 						"scopes": [
 							"text.jade",
-							"meta.tag.other entity.name.tag.jade"
+							"meta.tag.other",
+							"entity.name.tag.jade"
 						]
 					}
 				]
@@ -609,7 +610,8 @@
 						"value": "head",
 						"scopes": [
 							"text.jade",
-							"meta.tag.other entity.name.tag.jade"
+							"meta.tag.other",
+							"entity.name.tag.jade"
 						]
 					}
 				]

diff --git a/org.eclipse.tm4e.core.tests/src/test/resources/test-cases/themes/.gitignore b/org.eclipse.tm4e.core.tests/src/test/resources/test-cases/themes/.gitignore
@@ -0,0 +1 @@
+/tests/*.diff.html