vary canonical form behavior by language (#136)

* [en-*] stop using canonical form

* fix for de

* finish

3-tidy-up.js

@@ -140,7 +140,7 @@ function handleLine(line) {
     const parsedLine = JSON.parse(line);
     const { pos, sounds, forms } = parsedLine;
     if(!pos) return;
-    const word = getCanonicalForm(parsedLine);
+    const word = getCanonicalWordForm(parsedLine);
     if (!word) return;
     const readings = getReadings(word, parsedLine);
 
@@ -371,11 +371,26 @@ function processEnglishInflectionGlosses(glosses, word, pos) {
     }
 }
 
-function getCanonicalForm({word, forms}) {
+function getCanonicalWordForm({word, forms}) {
     if(!forms) return word;
 
-    const canonicalForm = forms.find(form => 
-        form.tags &&
+    switch(sourceIso) {
+        case 'ar':
+        case 'fa':
+        case 'la':
+        case 'ru':
+            return getCanonicalForm(word, forms); // canonical form is known to contain accent marks and such
+        case 'de':
+        // case 'fr': // canonical form sometimes just prepends the definite article, but many differ from the word in apostrophe variant. I don't know which is used in practice so leaving it until there's a yomitan preprocessor for french apostrophe usage. 
+        case 'en': 
+            return word; // canonical form is redundant, e.g. just prepends the definite article
+        default:
+            return getCanonicalForm(word, forms); // default could go either way. keeping existing behavior for now
+    }
+}
+
+function getCanonicalForm(word, forms) {
+    const canonicalForm = forms.find(form => form.tags &&
         form.tags.includes('canonical')
     );
     if (canonicalForm && canonicalForm.form) {

data/test/dict/en/en/tag_bank_1.json

@@ -26,5 +26,26 @@
     0,
     "figuratively",
     0
+  ],
+  [
+    "n",
+    "partOfSpeech",
+    -1,
+    "noun",
+    1
+  ],
+  [
+    "arch",
+    "archaism",
+    4,
+    "archaic",
+    -4
+  ],
+  [
+    "ltrry",
+    "",
+    0,
+    "literary",
+    0
   ]
 ]

data/test/dict/en/en/term_bank_1.json

@@ -49,5 +49,17 @@
     ],
     0,
     ""
+  ],
+  [
+    "wain",
+    "",
+    "n arch ltrry",
+    "n",
+    0,
+    [
+      "A wagon; a four-wheeled cart for hauling loads, usually pulled by horses or oxen."
+    ],
+    0,
+    ""
   ]
 ]

data/test/dict/en/en/term_bank_2.json

@@ -79,5 +79,22 @@
     ],
     0,
     ""
+  ],
+  [
+    "wains",
+    "",
+    "non-lemma",
+    "",
+    0,
+    [
+      [
+        "wain",
+        [
+          "plural"
+        ]
+      ]
+    ],
+    0,
+    ""
   ]
 ]

data/test/ipa/en/en/term_meta_bank_1.json

@@ -11,5 +11,18 @@
         }
       ]
     }
+  ],
+  [
+    "wain",
+    "ipa",
+    {
+      "reading": "wain",
+      "transcriptions": [
+        {
+          "ipa": "/weɪn/",
+          "tags": []
+        }
+      ]
+    }
   ]
 ]

data/test/tidy/en-en-forms-0.json

@@ -65,6 +65,28 @@
           ]
         ]
       }
+    ],
+    [
+      "wain",
+      {
+        "_type": "map",
+        "map": [
+          [
+            "wains",
+            {
+              "_type": "map",
+              "map": [
+                [
+                  "noun",
+                  [
+                    "plural"
+                  ]
+                ]
+              ]
+            }
+          ]
+        ]
+      }
     ]
   ]
 }

data/test/tidy/en-en-lemmas.json

@@ -64,5 +64,28 @@
         ]
       }
     }
+  },
+  "wain": {
+    "wain": {
+      "noun": {
+        "ipa": [
+          {
+            "ipa": "/weɪn/",
+            "tags": []
+          }
+        ],
+        "senses": [
+          {
+            "glosses": [
+              "(archaic or literary) A wagon; a four-wheeled cart for hauling loads, usually pulled by horses or oxen."
+            ],
+            "tags": [
+              "archaic",
+              "literary"
+            ]
+          }
+        ]
+      }
+    }
   }
 }