Merge pull request #32 from cs50/develop

v1.1.1

compare50/passes.py

@@ -12,6 +12,7 @@ class structure(Pass):
     preprocessors = [preprocessors.strip_whitespace,
                      preprocessors.strip_comments,
                      preprocessors.normalize_identifiers,
+                     preprocessors.normalize_builtin_types,
                      preprocessors.normalize_string_literals,
                      preprocessors.normalize_numeric_literals]
     comparator = comparators.Winnowing(k=25, t=35)

compare50/preprocessors.py

@@ -1,7 +1,7 @@
 import re
 
 import attr
-from pygments.token import Comment, Name, Number, String, Text
+from pygments.token import Comment, Name, Number, String, Text, Keyword
 
 from ._data import Token
 
@@ -17,6 +17,14 @@ def strip_whitespace(tokens):
             yield tok
 
 
+def normalize_builtin_types(tokens):
+    """Normalize builtin type names"""
+    for tok in tokens:
+        if tok.type in Keyword.Type:
+            tok.val = "t"
+        yield tok
+
+
 def strip_comments(tokens):
     """Remove all comments from tokens."""
     for tok in tokens:
@@ -36,9 +44,7 @@ def normalize_identifiers(tokens):
     for tok in tokens:
         if tok.type in Name:
             tok.val = "v"
-            yield tok
-        else:
-            yield tok
+        yield tok
 
 
 def normalize_string_literals(tokens):

setup.py

@@ -11,7 +11,7 @@
     ],
     license="GPLv3",
     description="This is compare50, with which you can compare files for similarities.",
-    install_requires=["attrs>=18,<20", "intervaltree>=2.1.0,<3", "lib50>=1.1.0,<2", "numpy>=1.15,<2", "pygments>=2.2,<3", "jinja2>=2.10,<3", "termcolor>=1.1.0,<2", "tqdm>=4.32,<5"],
+    install_requires=["attrs>=18,<20", "intervaltree>=2.1.0,<3", "lib50>=2,<3", "numpy>=1.15,<2", "pygments>=2.2,<3", "jinja2>=2.10,<3", "termcolor>=1.1.0,<2", "tqdm>=4.32,<5"],
     extras_require = {
         "develop": ["sphinx", "sphinx_rtd_theme", "line_profiler"]
     },
@@ -21,6 +21,6 @@
     packages=find_packages(exclude=["tests"]),
     scripts=["bin/compare50"],
     url="https://github.com/cs50/compare50",
-    version="1.1.0",
+    version="1.1.1",
     include_package_data=True,
 )