Translate vscode-textmate from TypeScript to Java to support

tokenizeLine2. See #38
eclipse-tm4e · May 11, 2017 · 4d27411 · 4d27411
1 parent 81fcb8d
commit 4d27411
Show file tree

Hide file tree

Showing 340 changed files with 1,185,464 additions and 396 deletions.
diff --git a/org.eclipse.tm4e.core.tests/pom.xml b/org.eclipse.tm4e.core.tests/pom.xml
@@ -7,4 +7,9 @@
     <artifactId>org.eclipse.tm4e</artifactId>
     <version>0.1.0-SNAPSHOT</version>
   </parent>
+
+  <properties>
+    <testSuite>${project.artifactId}</testSuite>
+    <testClass>org.eclipse.tm4e.core.AllTests</testClass>
+  </properties>
 </project>
diff --git a/org.eclipse.tm4e.core.tests/src/test/java/org/eclipse/tm4e/core/AllTests.java b/org.eclipse.tm4e.core.tests/src/test/java/org/eclipse/tm4e/core/AllTests.java
@@ -0,0 +1,15 @@
+package org.eclipse.tm4e.core;
+
+import org.eclipse.tm4e.core.grammar.VSCodeGrammarTest;
+import org.eclipse.tm4e.core.grammar.VSCodeTextMateTest;
+import org.eclipse.tm4e.core.theme.ThemeMatchingTest;
+import org.eclipse.tm4e.core.theme.ThemeParsingTest;
+import org.junit.runner.RunWith;
+import org.junit.runners.Suite;
+import org.junit.runners.Suite.SuiteClasses;
+
+@RunWith(Suite.class)
+@SuiteClasses({ VSCodeGrammarTest.class, VSCodeTextMateTest.class, ThemeMatchingTest.class, ThemeParsingTest.class })
+public class AllTests {
+
+}
diff --git a/...clipse.tm4e.core.tests/src/test/java/org/eclipse/tm4e/core/grammar/VSCodeGrammarTest.java b/...clipse.tm4e.core.tests/src/test/java/org/eclipse/tm4e/core/grammar/VSCodeGrammarTest.java
@@ -0,0 +1,116 @@
+/**
+ *  Copyright (c) 2015-2017 Angelo ZERR.
+ *  All rights reserved. This program and the accompanying materials
+ *  are made available under the terms of the Eclipse Public License v1.0
+ *  which accompanies this distribution, and is available at
+ *  http://www.eclipse.org/legal/epl-v10.html
+ *
+ *  Contributors:
+ *  Angelo Zerr <[email protected]> - initial API and implementation
+ */
+package org.eclipse.tm4e.core.grammar;
+
+import org.eclipse.tm4e.core.internal.grammar.StackElementMetadata;
+import org.eclipse.tm4e.core.internal.grammar.StandardTokenType;
+import org.eclipse.tm4e.core.theme.FontStyle;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * {@link StackElementMetadata} tests same than vscode-textmate.
+ * 
+ * @see https://github.com/Microsoft/vscode-textmate/blob/master/src/tests/grammar.test.ts
+ *
+ */
+public class VSCodeGrammarTest {
+
+	@Test
+	public void works() {
+		int value = StackElementMetadata.set(0, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101,
+				102);
+		assertEquals(value, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101, 102);
+	}
+
+	@Test
+	public void canOverwriteLanguageId() {
+		int value = StackElementMetadata.set(0, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101,
+				102);
+		assertEquals(value, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101, 102);
+
+		value = StackElementMetadata.set(value, 2, StandardTokenType.Other, FontStyle.NotSet, 0, 0);
+		assertEquals(value, 2, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101, 102);
+	}
+
+	@Test
+	public void canOverwriteTokenType() {
+		int value = StackElementMetadata.set(0, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101,
+				102);
+		assertEquals(value, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101, 102);
+
+		value = StackElementMetadata.set(value, 0, StandardTokenType.Comment, FontStyle.NotSet, 0, 0);
+		assertEquals(value, 1, StandardTokenType.Comment, FontStyle.Underline | FontStyle.Bold, 101, 102);
+	}
+
+	@Test
+	public void canOverwriteFontStyle() {
+		int value = StackElementMetadata.set(0, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101,
+				102);
+		assertEquals(value, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101, 102);
+
+		value = StackElementMetadata.set(value, 0, StandardTokenType.Other, FontStyle.None, 0, 0);
+		assertEquals(value, 1, StandardTokenType.RegEx, FontStyle.None, 101, 102);
+	}
+
+	@Test
+	public void canOverwriteForeground() {
+		int value = StackElementMetadata.set(0, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101,
+				102);
+		assertEquals(value, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101, 102);
+
+		value = StackElementMetadata.set(value, 0, StandardTokenType.Other, FontStyle.NotSet, 5, 0);
+		assertEquals(value, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 5, 102);
+	}
+
+	@Test
+	public void canOverwriteBackground() {
+		int value = StackElementMetadata.set(0, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101,
+				102);
+		assertEquals(value, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101, 102);
+
+		value = StackElementMetadata.set(value, 0, StandardTokenType.Other, FontStyle.NotSet, 0, 7);
+		assertEquals(value, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101, 7);
+	}
+
+	@Test
+	public void canWorkAtMaxValues() {
+		int maxLangId = 255;
+		int maxTokenType = StandardTokenType.Comment | StandardTokenType.Other | StandardTokenType.RegEx
+				| StandardTokenType.String;
+		int maxFontStyle = FontStyle.Bold | FontStyle.Italic | FontStyle.Underline;
+		int maxForeground = 511;
+		int maxBackground = 511;
+
+		int value = StackElementMetadata.set(0, maxLangId, maxTokenType, maxFontStyle, maxForeground, maxBackground);
+		assertEquals(value, maxLangId, maxTokenType, maxFontStyle, maxForeground, maxBackground);
+	}
+
+	private static void assertEquals(int metadata, int languageId, int tokenType, int fontStyle, int foreground, int background) {
+		String actual = "{\n" + 
+			"languageId: " + StackElementMetadata.getLanguageId(metadata) + ",\n" +
+			"tokenType: " + StackElementMetadata.getTokenType(metadata) + ",\n" +
+			"fontStyle: " + StackElementMetadata.getFontStyle(metadata) + ",\n" +
+			"foreground: " + StackElementMetadata.getForeground(metadata) + ",\n" +
+			"background: " + StackElementMetadata.getBackground(metadata) + ",\n" +			
+		"}";
+
+		String expected = "{\n" + 
+				"languageId: " + languageId + ",\n" +
+				"tokenType: " + tokenType + ",\n" +
+				"fontStyle: " + fontStyle + ",\n" +
+				"foreground: " + foreground + ",\n" +
+				"background: " + background + ",\n" +			
+			"}";
+
+		Assert.assertEquals("equals for " + StackElementMetadata.toBinaryStr(metadata), expected, actual);
+	}
+}
diff --git a/...core/grammar/test/VSCodeTextMateTest.java → ...tm4e/core/grammar/VSCodeTextMateTest.java b/...core/grammar/test/VSCodeTextMateTest.java → ...tm4e/core/grammar/VSCodeTextMateTest.java
@@ -8,7 +8,7 @@
  *  Contributors:
  *  Angelo Zerr <[email protected]> - initial API and implementation
  */
-package org.eclipse.tm4e.core.grammar.test;
+package org.eclipse.tm4e.core.grammar;
 
 import java.io.File;
 import java.io.FileReader;
@@ -17,6 +17,8 @@
 import java.util.Arrays;
 import java.util.List;
 
+import org.eclipse.tm4e.core.grammar.internal.MatcherTestImpl;
+import org.eclipse.tm4e.core.grammar.internal.RawTestImpl;
 import org.junit.runner.RunWith;
 import org.junit.runners.AllTests;
 
@@ -25,6 +27,13 @@
 
 import junit.framework.TestSuite;
 
+/**
+ * VSCode TextMate grammar tests which uses same vscode-textmate tests located
+ * at src\test\resources\test-cases
+ * 
+ * @see https://github.com/Microsoft/vscode-textmate/blob/master/src/tests/tests.ts
+ *
+ */
 @RunWith(AllTests.class)
 public class VSCodeTextMateTest {
 

diff --git a/...4e/core/grammar/test/MatcherTestImpl.java → ...ore/grammar/internal/MatcherTestImpl.java b/...4e/core/grammar/test/MatcherTestImpl.java → ...ore/grammar/internal/MatcherTestImpl.java
@@ -8,7 +8,7 @@
  *  Contributors:
  *  Angelo Zerr <[email protected]> - initial API and implementation
  */
-package org.eclipse.tm4e.core.grammar.test;
+package org.eclipse.tm4e.core.grammar.internal;
 
 import java.util.Collection;
 import java.util.List;

diff --git a/...e/tm4e/core/grammar/test/RawTestImpl.java → ...4e/core/grammar/internal/RawTestImpl.java b/...e/tm4e/core/grammar/test/RawTestImpl.java → ...4e/core/grammar/internal/RawTestImpl.java
@@ -8,7 +8,7 @@
  *  Contributors:
  *  Angelo Zerr <[email protected]> - initial API and implementation
  */
-package org.eclipse.tm4e.core.grammar.test;
+package org.eclipse.tm4e.core.grammar.internal;
 
 import java.io.File;
 import java.io.IOException;
@@ -23,7 +23,7 @@
 import org.eclipse.tm4e.core.grammar.ITokenizeLineResult;
 import org.eclipse.tm4e.core.grammar.StackElement;
 import org.eclipse.tm4e.core.logger.SystemLogger;
-import org.eclipse.tm4e.core.registry.IGrammarLocator;
+import org.eclipse.tm4e.core.registry.IRegistryOptions;
 import org.eclipse.tm4e.core.registry.Registry;
 import org.junit.Assert;
 import org.junit.runner.Describable;
@@ -84,7 +84,7 @@ public int countTestCases() {
 	}
 
 	private static void executeTest(RawTestImpl test, File testLocation) throws Exception {
-		IGrammarLocator locator = new IGrammarLocator() {
+		IRegistryOptions locator = new IRegistryOptions() {
 
 			@Override
 			public String getFilePath(String scopeName) {

diff --git a/...e/tm4e/core/grammar/test/RawTestLine.java → ...4e/core/grammar/internal/RawTestLine.java b/...e/tm4e/core/grammar/test/RawTestLine.java → ...4e/core/grammar/internal/RawTestLine.java
@@ -8,7 +8,7 @@
  *  Contributors:
  *  Angelo Zerr <[email protected]> - initial API and implementation
  */
-package org.eclipse.tm4e.core.grammar.test;
+package org.eclipse.tm4e.core.grammar.internal;
 
 import java.util.List;
 

diff --git a/...ipse/tm4e/core/grammar/test/RawToken.java → .../tm4e/core/grammar/internal/RawToken.java b/...ipse/tm4e/core/grammar/test/RawToken.java → .../tm4e/core/grammar/internal/RawToken.java
@@ -8,7 +8,7 @@
  *  Contributors:
  *  Angelo Zerr <[email protected]> - initial API and implementation
  */
-package org.eclipse.tm4e.core.grammar.test;
+package org.eclipse.tm4e.core.grammar.internal;
 
 import java.util.List;