Add operator-to-amslatex and builtin-constants...

operator-to-amslatex gives the AMSLaTeX string for unicode operator
builtin-constants tags characters which are Buitin Symbols that act as
nullary operators.

mathics_scanner/data/.gitignore

@@ -0,0 +1 @@
+/.python-version

mathics_scanner/data/named-characters.yml

@@ -32,6 +32,13 @@
 #                        equivalent that should replace other variants of it,
 #                        e.g. named character or ascii equivalent.
 #
+#   is-builtin-constant: If present, a boolean whether this is a Builtin Symbol as
+#                        defined by WMA that acts like a constant (or a nullary function).
+#                        Degree, I, Infinity, and PI are some examples such constants.
+#                        Usually, if this field is present it will be "true"; for "false"
+ #                       values (the majority), the field is omitted.
+#
+#
 #   is-letter-like: Whether or not this named-character is "letter-like", or
 #                   can be used as part of a symbol. For example \[alpha]
 #                   is letter-like. Therefore we can write: \[alpha]5 = "testing"
@@ -1986,6 +1993,7 @@ Decrement:
 Degree:
   esc-alias: deg
   has-unicode-inverse: false
+  is-builtin-constant: true
   is-letter-like: true
   unicode-equivalent: "\xB0"
   unicode-equivalent-name: DEGREE SIGN
@@ -3654,6 +3662,7 @@ ExponentialE:
   amslatex: "\\ExponentialE"
   esc-alias: ee
   has-unicode-inverse: true
+  is-builtin-constant: true
   is-letter-like: true
   unicode-equivalent: "\u2147"
   unicode-equivalent-name: DOUBLE-STRUCK ITALIC SMALL E
@@ -5854,6 +5863,7 @@ ImaginaryI:
   amslatex: "\\ComplexI"
   esc-alias: ii
   has-unicode-inverse: true
+  is-builtin-constant: true
   is-letter-like: true
   unicode-equivalent: "\u2148"
   unicode-equivalent-name: DOUBLE-STRUCK ITALIC SMALL I
@@ -5865,6 +5875,7 @@ ImaginaryJ:
   amslatex: "\\ComplexJ"
   esc-alias: jj
   has-unicode-inverse: true
+  is-builtin-constant: true
   is-letter-like: true
   unicode-equivalent: "\u2149"
   unicode-equivalent-name: DOUBLE-STRUCK ITALIC SMALL J
@@ -5905,6 +5916,7 @@ Infinity:
   amslatex: "\\infty"
   esc-alias: inf
   has-unicode-inverse: false
+  is-builtin-constant: true
   is-letter-like: true
   unicode-equivalent: "\u221E"
   unicode-equivalent-name: INFINITY
@@ -7751,6 +7763,7 @@ Pi:
   amslatex: "$\\pi$"
   esc-alias: p
   has-unicode-inverse: false
+  is-builtin-constant: true
   is-letter-like: false
   unicode-equivalent: "\u03C0"
   unicode-equivalent-name: GREEK SMALL LETTER PI

mathics_scanner/generate/build_operator_tables.py

@@ -7,7 +7,7 @@
 import sys
 from collections import defaultdict
 from pathlib import Path
-from typing import Dict
+from typing import Any, Dict, Tuple
 
 import click
 import yaml
@@ -55,25 +55,26 @@ def compile_tables(
         operator_precedence[k] = v["precedence"]
 
     box_operators = {}
-    flat_binary_operators = {}
-    left_binary_operators = {}
-    miscellaneous_operators = {}
+    flat_binary_operators: Dict[str, int] = {}
+    left_binary_operators: Dict[str, int] = {}
+    miscellaneous_operators: Dict[str, int] = {}
     no_meaning_infix_operators = {}
     no_meaning_postfix_operators = {}
     no_meaning_prefix_operators = {}
-    nonassoc_binary_operators = {}
+    nonassoc_binary_operators: Dict[str, int] = {}
     operator2string = defaultdict(list)
-    postfix_operators = {}
-    prefix_operators = {}
-    right_binary_operators = {}
-    ternary_operators = {}
+    operator2amslatex: Dict[str, str] = {}
+    postfix_operators: Dict[str, int] = {}
+    prefix_operators: Dict[str, int] = {}
+    right_binary_operators: Dict[str, int] = {}
+    ternary_operators: Dict[str, Tuple[int, int]] = {}
 
     for operator_name, operator_info in operator_data.items():
         precedence = operator_info["precedence"]
 
         affix = operator_info["affix"]
         arity = operator_info["arity"]
-        operator_dict = None
+        operator_dict: Dict[str, Any] = {}
 
         associativity = operator_info["associativity"]
         if arity == "Ternary":
@@ -118,6 +119,8 @@ def compile_tables(
 
         if unicode_char != "no-unicode":
             operator2string[operator_name].append(unicode_char)
+            if character_info.get("amslatex"):
+                operator2amslatex[unicode_char] = character_info["amslatex"]
         if ascii_chars != "no-ascii":
             operator2string[operator_name].append(ascii_chars)
 
@@ -148,6 +151,7 @@ def compile_tables(
         "no-meaning-postfix-operators": no_meaning_postfix_operators,
         "no-meaning-prefix-operators": no_meaning_prefix_operators,
         "non-associative-binary-operators": nonassoc_binary_operators,
+        "operator-to-amslatex": operator2amslatex,
         "operator-to_string": operator2string,
         "operator-precedence": operator_precedence,
         "postfix-operators": postfix_operators,

mathics_scanner/generate/build_tables.py

@@ -111,14 +111,14 @@ def compile_tables(data: dict) -> dict:
     # operator-to-unicode dictionary entry
     operator_to_precedence = {
         v["operator-name"]: v["precedence"]
-        for k, v in data.items()
+        for v in data.values()
         if "operator-name" in v and "precedence" in v
     }
 
     # operator-to-unicode dictionary entry
     operator_to_unicode = {
         v["operator-name"]: v.get("unicode-equivalent", v.get("ascii"))
-        for k, v in data.items()
+        for v in data.values()
         if "operator-name" in v and ("unicode-equivalent" in v or "ascii" in v)
     }
 
@@ -161,6 +161,12 @@ def compile_tables(data: dict) -> dict:
     ascii_operator_to_unicode = {}
     ascii_operator_to_wl_unicode = {}
 
+    builtin_constants = {
+        v["unicode-equivalent"]: k
+        for k, v in data.items()
+        if v.get("is-builtin-constant")
+    }
+
     for operator_name in operator_names:
         # Operators with ASCII sequences list entry
         v = data[operator_name]
@@ -181,7 +187,7 @@ def compile_tables(data: dict) -> dict:
     # unicode-to-operator dictionary entry
     unicode_to_operator = {
         v.get("unicode-equivalent", v.get("ascii")): v["operator-name"]
-        for k, v in data.items()
+        for v in data.values()
         if "operator-name" in v
     }
     # Conversion from WL to the fully qualified names dictionary entry
@@ -210,6 +216,7 @@ def compile_tables(data: dict) -> dict:
         "ascii-operator-to-character-symbol": ascii_operator_to_character_symbol,
         "ascii-operator-to-unicode": ascii_operator_to_unicode,
         "ascii-operator-to-wl-unicode": ascii_operator_to_wl_unicode,
+        "builtin-constants": builtin_constants,
         "letterlikes": letterlikes,
         "named-characters": named_characters,
         "operator-names": operator_names,

test/test_table_consistency.py → test/test_character_table_consistency.py

@@ -1,11 +1,10 @@
 # -*- coding: utf-8 -*-
 
-from mathics_scanner.characters import replace_wl_with_plain_text as wl_to_unicode
 from mathics_scanner.characters import replace_unicode_with_wl as unicode_to_wl
-
+from mathics_scanner.characters import replace_wl_with_plain_text as wl_to_unicode
 from mathics_scanner.load import (
-    load_mathics_character_yaml,
     load_mathics_character_json,
+    load_mathics_character_yaml,
 )
 
 yaml_data = load_mathics_character_yaml()
@@ -30,15 +29,15 @@ def test_roundtrip():
             if uni != wl:
                 assert (
                     uni == wl_to_unicode_dict[wl]
-                ), f"key {k} unicode {uni}, {wl_to_unicode[uni]}"
+                ), f"key {k} unicode {uni}, {wl_to_unicode(uni)}"
 
                 assert (
                     uni in unicode_to_wl_dict
                 ), f"key {k} has a non-trivial unicode inverse but isn't included in unicode-to-wl-dict"
 
                 assert (
                     unicode_to_wl_dict[uni] == wl
-                ), f"key {k} unicode {uni}, {wl_to_unicode[uni]}"
+                ), f"key {k} unicode {uni}, {wl_to_unicode(uni)}"
 
 
 def test_counts():

test/test_general_yaml_sanity.py

@@ -35,6 +35,7 @@ def test_yaml_field_names():
             "ascii",
             "esc-alias",
             "has-unicode-inverse",
+            "is-builtin-constant",
             "is-letter-like",
             "operator-name",
             "precedence",

test/test_operators.py

@@ -15,6 +15,8 @@
     operator_data = yaml.load(operator_f, Loader=yaml.FullLoader)
     character_data = yaml.load(character_f, Loader=yaml.FullLoader)
 
+operator_names = set(tuple(operator_data.keys()))
+
 
 def test_associativity_field():
     """
@@ -41,7 +43,6 @@ def test_operators():
             if "operator-name" in value
         ]
     )
-    operator_names = set(tuple(operator_data.keys()))
 
     left_character_operators = {
         operator_name
@@ -116,3 +117,14 @@ def test_meaningful_field():
                 "Postfix",
                 "Prefix",
             ), f"affix {affix} of {operator_name} not handled"
+
+
+def test_is_builtin_constant():
+    """
+    Check builtin constant names are operators (keys in the operator dictionary).
+    """
+    builtin_constant_names = [
+        name for name, info in character_data.items() if info.get("is_builtin_constant")
+    ]
+    for constant_name in builtin_constant_names:
+        assert constant_name in operator_names