feat: support parsing of lang cfg json files with trailing commas

eclipse-tm4e · Dec 13, 2023 · 7d41b71 · 7d41b71
1 parent 388292a
commit 7d41b71
Show file tree

Hide file tree

Showing 2 changed files with 69 additions and 2 deletions.
diff --git a/...ain/java/org/eclipse/tm4e/languageconfiguration/internal/model/LanguageConfiguration.java b/...ain/java/org/eclipse/tm4e/languageconfiguration/internal/model/LanguageConfiguration.java
@@ -8,6 +8,7 @@
  *
  * Contributors:
  * Lucas Bullen (Red Hat Inc.) - initial API and implementation
+ * Sebastian Thomschke (Vegard IT) - improve JSON parsing tolerance
  */
 package org.eclipse.tm4e.languageconfiguration.internal.model;
 
@@ -16,6 +17,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
 import org.eclipse.jdt.annotation.NonNull;
 import org.eclipse.jdt.annotation.NonNullByDefault;
@@ -38,6 +40,23 @@
  */
 public class LanguageConfiguration {
 
+	private static String removeTrailingCommas(String jsonString) {
+		/* matches:
+		 * --------------
+		 * },
+		 *    }
+		 * --------------
+		 * as well as:
+		 * --------------
+		 * },
+		 *   // foo
+		 *   // bar
+		 *    }
+		 * --------------
+		 */
+		return jsonString.replaceAll("(,)(\\s*\\n(\\s*\\/\\/.*\\n)*\\s*})", "$2");
+	}
+
 	/**
 	 * See JSON format at https://code.visualstudio.com/api/language-extensions/language-configuration-guide
 	 *
@@ -47,7 +66,28 @@ public class LanguageConfiguration {
 	@NonNullByDefault({})
 	@Nullable
 	public static LanguageConfiguration load(@NonNull final Reader reader) {
+		// GSON does not support trailing commas so we have to manually remove them -> maybe better switch to jackson json parser?
+		final var jsonString = removeTrailingCommas(new BufferedReader(reader).lines().collect(Collectors.joining("\n")));
 		return new GsonBuilder()
+				.registerTypeAdapter(String.class, (JsonDeserializer<String>) (json, typeOfT, context) -> {
+					if (json.isJsonObject()) {
+						/* for example:
+						 * "wordPattern": {
+						 *   "pattern": "...",
+						 *   "flags": "..."
+						 * },
+						 */
+						final var jsonObj = json.getAsJsonObject();
+						return jsonObj.has("pattern") && jsonObj.get("pattern").isJsonPrimitive() //
+								? jsonObj.get("pattern").getAsString()
+								: null;
+					}
+
+					/* for example:
+					 * "wordPattern": "...",
+					 */
+					return json.getAsString();
+				})
 
 				.registerTypeAdapter(OnEnterRule.class, (JsonDeserializer<OnEnterRule>) (json, typeOfT, context) -> {
 					if (!json.isJsonObject()) {
@@ -203,7 +243,7 @@ public static LanguageConfiguration load(@NonNull final Reader reader) {
 					return null;
 				})
 				.create()
-				.fromJson(new BufferedReader(reader), LanguageConfiguration.class);
+				.fromJson(jsonString, LanguageConfiguration.class);
 	}
 
 	@Nullable

diff --git a/...n/src/test/java/org/eclipse/tm4e/languageconfiguration/internal/supports/ParsingTest.java b/...n/src/test/java/org/eclipse/tm4e/languageconfiguration/internal/supports/ParsingTest.java
@@ -14,6 +14,13 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import org.eclipse.jdt.annotation.Nullable;
 import org.eclipse.tm4e.languageconfiguration.internal.model.LanguageConfiguration;
@@ -42,7 +49,6 @@ public void testCanLoadPhpLanguageConfig() throws Exception {
 		assertEquals(";:.,=}])>` \n\t", languageConfiguration.getAutoCloseBefore());
 		assertNotNull(languageConfiguration.getWordPattern());
 		assertNotNull(languageConfiguration.getOnEnterRules());
-
 		assertNotNull(languageConfiguration.getSurroundingPairs());
 		assertNotNull(languageConfiguration.getFolding());
 	}
@@ -63,4 +69,25 @@ public void testCanLoadRustLanguageConfig() throws Exception {
 		assertNotNull(languageConfiguration.getSurroundingPairs());
 		assertNull(languageConfiguration.getFolding());
 	}
+
+	@Test
+	void testLanguagePackLangConfigs() throws IOException {
+		final var count = new AtomicInteger();
+		Files.walkFileTree(Paths.get("../org.eclipse.tm4e.language_pack"), new SimpleFileVisitor<Path>() {
+			@Override
+			public FileVisitResult visitFile(final Path file, final @Nullable BasicFileAttributes attrs) throws IOException {
+				if (file.getFileName().toString().endsWith("language-configuration.json")) {
+					try (var input = Files.newBufferedReader(file)) {
+						System.out.println("Parsing [" + file + "]...");
+						final var languageConfiguration = LanguageConfiguration.load(input);
+						count.incrementAndGet();
+						assertNotNull(languageConfiguration);
+					}
+				}
+				return FileVisitResult.CONTINUE;
+			}
+		});
+		System.out.println("Successfully parsed " + count.intValue() + " language configurations.");
+		assertTrue(count.intValue() > 10, "Only " + count.intValue() + " language configurations found, expected more than 10!");
+	}
 }