Better suggestion with diacritics (#1223)

* Update cdict

* scripts/subst_of_compose.py: Compute substitutions

from compose mappings. They are used when building dictionaries.

* Add substitutions compose data

* Better suggestion with diacritics

This improves the suggestions for words that contain diacritics and
uppercase letters.

This works by stripping diacritics both when building the dictionaries
(using word aliases added in cdict: https://github.com/Julow/cdict/pull/3)
and during lookup. Cdict then takes care of resolving the correct word.

The substitutions are generated using mappings from `fn`, `shift` and
all the `accent_*` modifiers into srcs/compose/substitutions.json
This can be updated easily when more mappings are added.

scripts/subst_of_compose.py

@@ -0,0 +1,90 @@
+# Create a substitution file for cdict from the various compose mappings.
+# This is used when building the dictionaries and when making word suggestions
+# disregarding case and diacritics.
+
+import sys, os, json, glob, unicodedata
+
+OUTPUT_FILE = "srcs/compose/substitutions.json"
+
+def warn(msg):
+    print("Warning: " + msg, file=sys.stderr)
+
+# From srcs/compose/compile.py
+def strip_cstyle_comments(inp):
+    def strip_line(line):
+        i = line.find("//")
+        return line[:i] + "\n" if i >= 0 else line
+    return "".join(map(strip_line, inp))
+
+def parse(fname):
+    with open(fname, "r") as inp:
+        return json.loads(strip_cstyle_comments(inp))
+
+def is_char16(c):
+    return len(c) == 1 and ord(c) < 65536
+
+def get_mappings(tree):
+    for c, r in tree.items():
+        # Remove deep compose sequences and remove mappings to non-char keys or
+        # to characters that do not fit in a Java 16-bit char.
+        if isinstance(r, str) and is_char16(r) and is_char16(c):
+            yield c, r
+
+def mappings_from_compose_files():
+    for f in glob.glob("srcs/compose/*.json"):
+        if f == OUTPUT_FILE:
+            continue
+        yield from get_mappings(parse(f))
+
+# The definition of shift doesn't contain any letters as shift is implemented
+# using Java's API so we generate it using Python's API. It's not important if
+# both are not equivalent.
+def add_case_variants(mappings):
+    for c in "abcdefghijklmnopqrstuvwxyz":
+        yield c, c.upper()
+    for c, r in mappings:
+        c_low = c.lower()
+        if c_low != c and is_char16(c_low): yield c_low, r
+        r_up = r.upper()
+        if r_up != r and is_char16(r_up): yield c, r_up
+        yield c, r
+
+# Remove unecessary characters to reduce the lookup time
+ALLOWED_CAT = [ "Ll", "Lu", "Lt", "Lo" ]
+def remove_non_letters(mappings):
+    for c, r in mappings:
+        cat = unicodedata.category(c)
+        if cat in ALLOWED_CAT:
+            yield c, r
+
+def resolve_mappings(mappings):
+    m = {}
+    # Sort mappings to keep the lowest char in case of a conflict
+    for c, r in sorted(mappings, key=lambda it: it[1]):
+        if r in m:
+            if m[r] != c:
+                warn("Conflicting mapping '%s -> %s' and '%s -> %s'" %
+                      (c, r, m[r], r))
+            continue
+        m[r] = c
+    def resolve(c, trace=None):
+        if c in m:
+            if trace is None:
+                trace = set()
+            elif c in trace:
+                return c
+            trace.add(c)
+            return resolve(m[c], trace=trace)
+        return c
+    return { r: resolve(c) for r, c in m.items() }
+
+
+with open(OUTPUT_FILE, "w") as out:
+    json.dump(
+            resolve_mappings(
+                add_case_variants(
+                    remove_non_letters(
+                        mappings_from_compose_files()))),
+              out, ensure_ascii=False, indent=2)
+
+print("Generated " + OUTPUT_FILE)

srcs/compose/compile.py

@@ -198,6 +198,8 @@ def make_automata(tries):
             states.append((None, None))
         # Add nested nodes and fill the current node
         for c in sorted(t.keys()):
+            if len(c) > 1 or ord(c[0]) > 65535:
+                raise Exception("Char out of range: " + c)
             states[i] = (c, add_node(t[c]))
             i += 1
         return this_node_index

srcs/compose/substitutions.json

@@ -0,0 +1,949 @@
+{
+  "A": "a",
+  "B": "b",
+  "C": "c",
+  "D": "d",
+  "E": "e",
+  "F": "f",
+  "G": "g",
+  "H": "h",
+  "I": "i",
+  "J": "j",
+  "K": "k",
+  "L": "l",
+  "M": "m",
+  "N": "n",
+  "O": "o",
+  "P": "p",
+  "Q": "q",
+  "R": "r",
+  "S": "s",
+  "T": "t",
+  "U": "u",
+  "V": "v",
+  "W": "w",
+  "X": "x",
+  "Y": "y",
+  "Z": "z",
+  "x": "x",
+  "¢": "c",
+  "£": "l",
+  "¥": "y",
+  "ª": "a",
+  "µ": "u",
+  "º": "o",
+  "À": "a",
+  "Á": "a",
+  "Â": "a",
+  "Ã": "a",
+  "Ä": "a",
+  "Å": "a",
+  "Æ": "a",
+  "Ç": "c",
+  "È": "e",
+  "É": "e",
+  "Ê": "e",
+  "Ë": "e",
+  "Ì": "i",
+  "Í": "i",
+  "Î": "i",
+  "Ï": "i",
+  "Ñ": "n",
+  "Ò": "o",
+  "Ó": "o",
+  "Ô": "o",
+  "Õ": "o",
+  "Ö": "o",
+  "Ø": "o",
+  "Ù": "u",
+  "Ú": "u",
+  "Û": "u",
+  "Ü": "u",
+  "Ý": "y",
+  "à": "a",
+  "á": "a",
+  "â": "a",
+  "ã": "a",
+  "ä": "a",
+  "å": "a",
+  "æ": "a",
+  "ç": "c",
+  "è": "e",
+  "é": "e",
+  "ê": "e",
+  "ë": "e",
+  "ì": "i",
+  "í": "i",
+  "î": "i",
+  "ï": "i",
+  "ñ": "n",
+  "ò": "o",
+  "ó": "o",
+  "ô": "o",
+  "õ": "o",
+  "ö": "o",
+  "ø": "o",
+  "ù": "u",
+  "ú": "u",
+  "û": "u",
+  "ü": "u",
+  "ý": "y",
+  "ÿ": "y",
+  "Ā": "a",
+  "ā": "a",
+  "Ą": "a",
+  "ą": "a",
+  "Ć": "c",
+  "ć": "c",
+  "Ĉ": "c",
+  "ĉ": "c",
+  "Ċ": "c",
+  "ċ": "c",
+  "Č": "c",
+  "č": "c",
+  "Ď": "d",
+  "ď": "d",
+  "Đ": "d",
+  "đ": "d",
+  "Ē": "e",
+  "ē": "e",
+  "Ė": "e",
+  "ė": "e",
+  "Ę": "e",
+  "ę": "e",
+  "Ě": "e",
+  "ě": "e",
+  "Ĝ": "g",
+  "ĝ": "g",
+  "Ġ": "g",
+  "ġ": "g",
+  "Ģ": "g",
+  "ģ": "g",
+  "Ĥ": "h",
+  "ĥ": "h",
+  "Ħ": "h",
+  "ħ": "h",
+  "Ĩ": "i",
+  "ĩ": "i",
+  "Ī": "i",
+  "ī": "i",
+  "Į": "i",
+  "į": "i",
+  "İ": "i",
+  "ı": "i",
+  "Ĵ": "j",
+  "ĵ": "j",
+  "Ķ": "k",
+  "ķ": "k",
+  "Ĺ": "l",
+  "ĺ": "l",
+  "Ļ": "l",
+  "ļ": "l",
+  "Ľ": "l",
+  "ľ": "l",
+  "Ł": "l",
+  "ł": "l",
+  "Ń": "n",
+  "ń": "n",
+  "Ņ": "n",
+  "ņ": "n",
+  "Ň": "n",
+  "ň": "n",
+  "Ō": "o",
+  "ō": "o",
+  "Ő": "o",
+  "ő": "o",
+  "Œ": "o",
+  "œ": "o",
+  "Ŕ": "r",
+  "ŕ": "r",
+  "Ŗ": "r",
+  "ŗ": "r",
+  "Ř": "r",
+  "ř": "r",
+  "Ś": "s",
+  "ś": "s",
+  "Ŝ": "ŝ",
+  "ŝ": "ŝ",
+  "Ş": "s",
+  "ş": "s",
+  "Š": "s",
+  "š": "s",
+  "Ţ": "t",
+  "ţ": "t",
+  "Ť": "t",
+  "ť": "t",
+  "Ŧ": "t",
+  "ŧ": "t",
+  "Ũ": "u",
+  "ũ": "u",
+  "Ū": "u",
+  "ū": "u",
+  "Ů": "u",
+  "ů": "u",
+  "Ű": "u",
+  "ű": "u",
+  "Ų": "u",
+  "ų": "u",
+  "Ŵ": "w",
+  "ŵ": "w",
+  "Ŷ": "y",
+  "ŷ": "y",
+  "Ÿ": "y",
+  "Ź": "z",
+  "ź": "z",
+  "Ż": "z",
+  "ż": "z",
+  "Ž": "z",
+  "ž": "z",
+  "ƀ": "b",
+  "Ɣ": "γ",
+  "Ɨ": "i",
+  "ƚ": "l",
+  "Ɵ": "o",
+  "Ơ": "o",
+  "ơ": "o",
+  "Ʀ": "r",
+  "Ư": "u",
+  "ư": "u",
+  "Ƶ": "z",
+  "ƶ": "z",
+  "Ǎ": "a",
+  "ǎ": "a",
+  "Ǐ": "i",
+  "ǐ": "i",
+  "Ǒ": "o",
+  "ǒ": "o",
+  "Ǔ": "u",
+  "ǔ": "u",
+  "Ǖ": "u",
+  "ǖ": "u",
+  "Ǘ": "u",
+  "ǘ": "u",
+  "Ǚ": "u",
+  "ǚ": "u",
+  "Ǜ": "u",
+  "ǜ": "u",
+  "Ǟ": "a",
+  "ǟ": "a",
+  "Ǡ": "a",
+  "ǡ": "a",
+  "Ǣ": "a",
+  "ǣ": "a",
+  "Ǥ": "g",
+  "ǥ": "g",
+  "Ǧ": "g",
+  "ǧ": "g",
+  "Ǩ": "k",
+  "ǩ": "k",
+  "Ǫ": "o",
+  "ǫ": "o",
+  "Ǭ": "o",
+  "ǭ": "o",
+  "Ǯ": "ʒ",
+  "ǯ": "ʒ",
+  "ǰ": "j",
+  "Ǵ": "g",
+  "ǵ": "g",
+  "Ǹ": "n",
+  "ǹ": "n",
+  "Ǻ": "a",
+  "ǻ": "a",
+  "Ǽ": "a",
+  "ǽ": "a",
+  "Ǿ": "o",
+  "ǿ": "o",
+  "Ȁ": "a",
+  "ȁ": "a",
+  "Ȅ": "e",
+  "ȅ": "e",
+  "Ȉ": "i",
+  "ȉ": "i",
+  "Ȍ": "o",
+  "ȍ": "o",
+  "Ȑ": "r",
+  "ȑ": "r",
+  "Ȕ": "u",
+  "ȕ": "u",
+  "Ȟ": "h",
+  "ȟ": "h",
+  "Ȧ": "a",
+  "ȧ": "a",
+  "Ȩ": "e",
+  "ȩ": "e",
+  "Ȫ": "o",
+  "ȫ": "o",
+  "Ȭ": "o",
+  "ȭ": "o",
+  "Ȯ": "o",
+  "ȯ": "o",
+  "Ȱ": "o",
+  "ȱ": "o",
+  "Ȳ": "y",
+  "ȳ": "y",
+  "ȷ": "j",
+  "Ⱥ": "a",
+  "Ȼ": "c",
+  "ȼ": "c",
+  "Ƚ": "l",
+  "Ⱦ": "t",
+  "Ƀ": "b",
+  "Ʉ": "u",
+  "Ɇ": "e",
+  "ɇ": "e",
+  "Ɉ": "j",
+  "ɉ": "j",
+  "Ɍ": "r",
+  "ɍ": "r",
+  "Ɏ": "y",
+  "ɏ": "y",
+  "ɟ": "j",
+  "ɢ": "g",
+  "ɣ": "γ",
+  "ɨ": "i",
+  "ɪ": "i",
+  "ɴ": "n",
+  "ɵ": "o",
+  "ɶ": "o",
+  "ʀ": "r",
+  "ʉ": "u",
+  "ʏ": "y",
+  "ʙ": "b",
+  "ʛ": "ɠ",
+  "ʜ": "h",
+  "ʟ": "l",
+  "ʰ": "h",
+  "ʱ": "ɦ",
+  "ʲ": "j",
+  "ʳ": "r",
+  "ʴ": "ɹ",
+  "ʵ": "ɻ",
+  "ʶ": "ʁ",
+  "ʷ": "w",
+  "ʸ": "y",
+  "ˠ": "γ",
+  "ˡ": "l",
+  "ˢ": "s",
+  "ˣ": "x",
+  "Ά": "α",
+  "Έ": "ε",
+  "Ή": "η",
+  "Ί": "ι",
+  "Ό": "ο",
+  "Ύ": "υ",
+  "Β": "β",
+  "Ε": "ε",
+  "Θ": "θ",
+  "Κ": "κ",
+  "Μ": "u",
+  "Π": "π",
+  "Ρ": "ρ",
+  "Σ": "σ",
+  "Φ": "φ",
+  "Ϊ": "ι",
+  "Ϋ": "υ",
+  "ά": "α",
+  "έ": "ε",
+  "ή": "η",
+  "ί": "ι",
+  "ΰ": "υ",
+  "ς": "σ",
+  "ϊ": "ι",
+  "ϋ": "υ",
+  "ό": "ο",
+  "ύ": "υ",
+  "ϐ": "β",
+  "ϑ": "θ",
+  "ϒ": "υ",
+  "ϔ": "υ",
+  "ϕ": "φ",
+  "ϖ": "π",
+  "ϰ": "κ",
+  "ϱ": "ρ",
+  "ϴ": "θ",
+  "ϵ": "ε",
+  "Ѐ": "е",
+  "Ђ": "ꙉ",
+  "Ѓ": "г",
+  "Є": "э",
+  "Ѕ": "с",
+  "І": "и",
+  "Ј": "й",
+  "Ћ": "ч",
+  "Ќ": "к",
+  "Ѝ": "и",
+  "Џ": "ҷ",
+  "И": "і",
+  "Й": "ј",
+  "Ч": "ћ",
+  "Ъ": "ь",
+  "Ы": "ꙑ",
+  "Ь": "ъ",
+  "Я": "ꙗ",
+  "и": "і",
+  "й": "ј",
+  "ч": "ћ",
+  "ъ": "ь",
+  "ы": "ꙑ",
+  "ь": "ъ",
+  "я": "ꙗ",
+  "ѐ": "е",
+  "ђ": "ꙉ",
+  "ѓ": "г",
+  "є": "э",
+  "ѕ": "с",
+  "і": "и",
+  "ј": "й",
+  "ћ": "ч",
+  "ќ": "к",
+  "ѝ": "и",
+  "џ": "ҷ",
+  "Ѣ": "е",
+  "ѣ": "е",
+  "Ѧ": "н",
+  "ѧ": "н",
+  "Ѫ": "м",
+  "ѫ": "м",
+  "Ѱ": "ш",
+  "ѱ": "ш",
+  "Ѳ": "ф",
+  "ѳ": "ф",
+  "Ѷ": "ѵ",
+  "ѷ": "ѵ",
+  "Ѻ": "о",
+  "ѻ": "о",
+  "Ҁ": "қ",
+  "ҁ": "қ",
+  "Ҍ": "ь",
+  "ҍ": "ь",
+  "Ґ": "г",
+  "ґ": "г",
+  "Ҙ": "д",
+  "ҙ": "д",
+  "Қ": "ҁ",
+  "қ": "ҁ",
+  "Ұ": "ү",
+  "ұ": "ү",
+  "Ҷ": "џ",
+  "ҷ": "џ",
+  "Һ": "х",
+  "һ": "х",
+  "Ӓ": "а",
+  "ӓ": "а",
+  "Ӛ": "ә",
+  "ӛ": "ә",
+  "Ӝ": "ж",
+  "ӝ": "ж",
+  "Ӟ": "з",
+  "ӟ": "з",
+  "Ӣ": "и",
+  "ӣ": "и",
+  "Ӥ": "и",
+  "ӥ": "и",
+  "Ӧ": "о",
+  "ӧ": "о",
+  "Ө": "о",
+  "ө": "о",
+  "Ӫ": "о",
+  "ӫ": "о",
+  "Ӭ": "э",
+  "ӭ": "э",
+  "Ӯ": "у",
+  "ӯ": "у",
+  "Ӱ": "у",
+  "ӱ": "у",
+  "Ӳ": "у",
+  "ӳ": "у",
+  "Ӵ": "ч",
+  "ӵ": "ч",
+  "Ӹ": "ы",
+  "ӹ": "ы",
+  "Ӿ": "х",
+  "ӿ": "х",
+  "Ԩ": "ӈ",
+  "ԩ": "ӈ",
+  "Ԯ": "л",
+  "ԯ": "л",
+  "ء": "ع",
+  "آ": "ا",
+  "ذ": "د",
+  "ص": "س",
+  "ض": "س",
+  "ط": "ت",
+  "ظ": "ز",
+  "غ": "ق",
+  "ـ": "م",
+  "ك": "ک",
+  "ه": "ھ",
+  "ي": "ی",
+  "ٮ": "ب",
+  "پ": "ث",
+  "ڕ": "ر",
+  "ژ": "ز",
+  "ڡ": "و",
+  "ڤ": "ف",
+  "ک": "ك",
+  "گ": "ق",
+  "ڵ": "ل",
+  "ں": "ن",
+  "ھ": "ه",
+  "ہ": "ح",
+  "ی": "ي",
+  "ं": "म",
+  "ः": "ह",
+  "आ": "अ",
+  "ई": "इ",
+  "ऊ": "उ",
+  "ऍ": "ए",
+  "ऎ": "ए",
+  "ऐ": "ए",
+  "ऑ": "अ",
+  "ऒ": "ओ",
+  "औ": "ओ",
+  "ख": "क",
+  "घ": "ग",
+  "छ": "च",
+  "झ": "ज",
+  "ठ": "ट",
+  "ढ": "ड",
+  "ण": "न",
+  "थ": "त",
+  "ध": "द",
+  "ऩ": "न",
+  "भ": "ब",
+  "ऱ": "र",
+  "ळ": "ल",
+  "ऴ": "ल",
+  "श": "स",
+  "क़": "क",
+  "ख़": "क",
+  "ग़": "ग",
+  "ज़": "ज",
+  "ड़": "ड",
+  "ढ़": "ड",
+  "फ़": "फ",
+  "य़": "य",
+  "ॠ": "ऋ",
+  "ॡ": "ऌ",
+  "ॲ": "अ",
+  "ॳ": "इ",
+  "ॴ": "इ",
+  "ॵ": "ओ",
+  "ॶ": "उ",
+  "ॷ": "उ",
+  "ॸ": "ट",
+  "ॹ": "ज",
+  "ॺ": "व",
+  "ॻ": "ग",
+  "ॼ": "च",
+  "ॽ": "ऽ",
+  "ॾ": "न",
+  "ॿ": "ब",
+  "ં": "મ",
+  "ઃ": "હ",
+  "આ": "અ",
+  "ઈ": "ઇ",
+  "ઊ": "ઉ",
+  "ઐ": "એ",
+  "ઔ": "ઓ",
+  "ખ": "ક",
+  "ઘ": "ગ",
+  "છ": "ચ",
+  "ઝ": "જ",
+  "ઠ": "ટ",
+  "ઢ": "ડ",
+  "ણ": "ન",
+  "થ": "ત",
+  "ધ": "દ",
+  "ફ": "પ",
+  "ભ": "બ",
+  "ળ": "લ",
+  "શ": "સ",
+  "ᴀ": "a",
+  "ᴁ": "a",
+  "ᴃ": "b",
+  "ᴄ": "c",
+  "ᴅ": "d",
+  "ᴆ": "ð",
+  "ᴇ": "e",
+  "ᴊ": "j",
+  "ᴋ": "k",
+  "ᴌ": "l",
+  "ᴍ": "m",
+  "ᴎ": "ŋ",
+  "ᴏ": "o",
+  "ᴐ": "ɔ",
+  "ᴕ": "ȣ",
+  "ᴘ": "p",
+  "ᴙ": "я",
+  "ᴚ": "ɹ",
+  "ᴛ": "t",
+  "ᴜ": "u",
+  "ᴠ": "v",
+  "ᴡ": "w",
+  "ᴢ": "z",
+  "ᴣ": "ʒ",
+  "ᴦ": "γ",
+  "ᴧ": "λ",
+  "ᴨ": "π",
+  "ᴩ": "ρ",
+  "ᴪ": "ψ",
+  "ᴫ": "л",
+  "ᵃ": "a",
+  "ᵄ": "ɐ",
+  "ᵆ": "ᴂ",
+  "ᵇ": "b",
+  "ᵈ": "d",
+  "ᵉ": "e",
+  "ᵊ": "ə",
+  "ᵋ": "ɛ",
+  "ᵌ": "ᴈ",
+  "ᵍ": "g",
+  "ᵎ": "ᴉ",
+  "ᵏ": "k",
+  "ᵐ": "m",
+  "ᵑ": "ŋ",
+  "ᵒ": "o",
+  "ᵓ": "ɔ",
+  "ᵖ": "p",
+  "ᵗ": "t",
+  "ᵘ": "u",
+  "ᵙ": "ᴝ",
+  "ᵚ": "ɯ",
+  "ᵛ": "v",
+  "ᵝ": "β",
+  "ᵟ": "δ",
+  "ᵠ": "φ",
+  "ᵡ": "χ",
+  "ᵢ": "i",
+  "ᵣ": "r",
+  "ᵤ": "u",
+  "ᵥ": "v",
+  "ᵦ": "β",
+  "ᵧ": "γ",
+  "ᵨ": "ρ",
+  "ᵩ": "φ",
+  "ᵪ": "χ",
+  "ᵻ": "i",
+  "ᵽ": "p",
+  "ᵾ": "u",
+  "ᶛ": "ɒ",
+  "ᶜ": "c",
+  "ᶝ": "ɕ",
+  "ᶟ": "ɜ",
+  "ᶠ": "f",
+  "ᶡ": "j",
+  "ᶣ": "ɥ",
+  "ᶤ": "i",
+  "ᶥ": "ι",
+  "ᶬ": "ɱ",
+  "ᶭ": "ɰ",
+  "ᶱ": "o",
+  "ᶳ": "ʂ",
+  "ᶴ": "ʃ",
+  "ᶶ": "u",
+  "ᶷ": "ʊ",
+  "ᶺ": "ʌ",
+  "ᶻ": "z",
+  "ᶾ": "ʒ",
+  "ᶿ": "θ",
+  "Ḃ": "b",
+  "ḃ": "b",
+  "Ḅ": "b",
+  "ḅ": "b",
+  "Ḉ": "c",
+  "ḉ": "c",
+  "Ḋ": "d",
+  "ḋ": "d",
+  "Ḍ": "d",
+  "ḍ": "d",
+  "Ḑ": "d",
+  "ḑ": "d",
+  "Ḕ": "e",
+  "ḕ": "e",
+  "Ḗ": "e",
+  "ḗ": "e",
+  "Ḝ": "ĕ",
+  "ḝ": "ĕ",
+  "Ḟ": "f",
+  "ḟ": "f",
+  "Ḡ": "g",
+  "ḡ": "g",
+  "Ḣ": "h",
+  "ḣ": "h",
+  "Ḥ": "h",
+  "ḥ": "h",
+  "Ḧ": "h",
+  "ḧ": "h",
+  "Ḩ": "h",
+  "ḩ": "h",
+  "Ḯ": "i",
+  "ḯ": "i",
+  "Ḱ": "k",
+  "ḱ": "k",
+  "Ḳ": "k",
+  "ḳ": "k",
+  "Ḷ": "l",
+  "ḷ": "l",
+  "Ḹ": "l",
+  "ḹ": "l",
+  "Ḿ": "m",
+  "ḿ": "m",
+  "Ṁ": "m",
+  "ṁ": "m",
+  "Ṃ": "m",
+  "ṃ": "m",
+  "Ṅ": "n",
+  "ṅ": "n",
+  "Ṇ": "n",
+  "ṇ": "n",
+  "Ṍ": "o",
+  "ṍ": "o",
+  "Ṏ": "o",
+  "ṏ": "o",
+  "Ṑ": "o",
+  "ṑ": "o",
+  "Ṓ": "o",
+  "ṓ": "o",
+  "Ṕ": "p",
+  "ṕ": "p",
+  "Ṗ": "p",
+  "ṗ": "p",
+  "Ṙ": "r",
+  "ṙ": "r",
+  "Ṛ": "r",
+  "ṛ": "r",
+  "Ṝ": "r",
+  "ṝ": "r",
+  "Ṡ": "s",
+  "ṡ": "s",
+  "Ṣ": "s",
+  "ṣ": "s",
+  "Ṥ": "s",
+  "ṥ": "s",
+  "Ṧ": "s",
+  "ṧ": "s",
+  "Ṩ": "s",
+  "ṩ": "s",
+  "Ṫ": "t",
+  "ṫ": "t",
+  "Ṭ": "t",
+  "ṭ": "t",
+  "Ṹ": "u",
+  "ṹ": "u",
+  "Ṽ": "v",
+  "ṽ": "v",
+  "Ṿ": "v",
+  "ṿ": "v",
+  "Ẁ": "w",
+  "ẁ": "w",
+  "Ẃ": "w",
+  "ẃ": "w",
+  "Ẅ": "w",
+  "ẅ": "w",
+  "Ẇ": "w",
+  "ẇ": "w",
+  "Ẉ": "w",
+  "ẉ": "w",
+  "Ẋ": "x",
+  "ẋ": "x",
+  "Ẍ": "x",
+  "ẍ": "x",
+  "Ẏ": "y",
+  "ẏ": "y",
+  "Ẑ": "z",
+  "ẑ": "z",
+  "Ẓ": "z",
+  "ẓ": "z",
+  "ẗ": "t",
+  "ẘ": "w",
+  "ẙ": "y",
+  "ẛ": "ſ",
+  "ẞ": "ß",
+  "Ạ": "a",
+  "ạ": "a",
+  "Ả": "a",
+  "ả": "a",
+  "Ấ": "a",
+  "ấ": "a",
+  "Ầ": "a",
+  "ầ": "a",
+  "Ẩ": "a",
+  "ẩ": "a",
+  "Ẫ": "a",
+  "ẫ": "a",
+  "Ậ": "a",
+  "ậ": "a",
+  "Ắ": "ă",
+  "ắ": "ă",
+  "Ằ": "ă",
+  "ằ": "ă",
+  "Ẳ": "ă",
+  "ẳ": "ă",
+  "Ẵ": "ă",
+  "ẵ": "ă",
+  "Ặ": "ă",
+  "ặ": "ă",
+  "Ẹ": "e",
+  "ẹ": "e",
+  "Ẻ": "e",
+  "ẻ": "e",
+  "Ẽ": "e",
+  "ẽ": "e",
+  "Ế": "e",
+  "ế": "e",
+  "Ề": "e",
+  "ề": "e",
+  "Ể": "e",
+  "ể": "e",
+  "Ễ": "e",
+  "ễ": "e",
+  "Ệ": "e",
+  "ệ": "e",
+  "Ỉ": "i",
+  "ỉ": "i",
+  "Ị": "i",
+  "ị": "i",
+  "Ọ": "o",
+  "ọ": "o",
+  "Ỏ": "o",
+  "ỏ": "o",
+  "Ố": "o",
+  "ố": "o",
+  "Ồ": "o",
+  "ồ": "o",
+  "Ổ": "o",
+  "ổ": "o",
+  "Ỗ": "o",
+  "ỗ": "o",
+  "Ộ": "o",
+  "ộ": "o",
+  "Ớ": "o",
+  "ớ": "o",
+  "Ờ": "o",
+  "ờ": "o",
+  "Ở": "o",
+  "ở": "o",
+  "Ỡ": "o",
+  "ỡ": "o",
+  "Ợ": "o",
+  "ợ": "o",
+  "Ụ": "u",
+  "ụ": "u",
+  "Ủ": "u",
+  "ủ": "u",
+  "Ứ": "u",
+  "ứ": "u",
+  "Ừ": "u",
+  "ừ": "u",
+  "Ử": "u",
+  "ử": "u",
+  "Ữ": "u",
+  "ữ": "u",
+  "Ự": "u",
+  "ự": "u",
+  "Ỳ": "y",
+  "ỳ": "y",
+  "Ỵ": "y",
+  "ỵ": "y",
+  "Ỷ": "y",
+  "ỷ": "y",
+  "Ỹ": "y",
+  "ỹ": "y",
+  "ὰ": "α",
+  "ὲ": "ε",
+  "ὴ": "η",
+  "ὶ": "ι",
+  "ὸ": "ο",
+  "ὺ": "υ",
+  "ὼ": "ω",
+  "ᾱ": "α",
+  "Ᾱ": "α",
+  "Ὰ": "α",
+  "Ὲ": "ε",
+  "Ὴ": "η",
+  "ῑ": "ι",
+  "Ῑ": "ι",
+  "Ὶ": "ι",
+  "ῡ": "υ",
+  "ῢ": "υ",
+  "ῧ": "ῦ",
+  "Ῡ": "υ",
+  "Ὺ": "υ",
+  "Ὸ": "ο",
+  "Ὼ": "ω",
+  "ⁱ": "i",
+  "ⁿ": "n",
+  "ₐ": "a",
+  "ₑ": "e",
+  "ₒ": "o",
+  "ₓ": "x",
+  "ₔ": "ə",
+  "ₕ": "h",
+  "ₖ": "k",
+  "ₗ": "l",
+  "ₘ": "m",
+  "ₙ": "n",
+  "ₚ": "p",
+  "ₛ": "s",
+  "ₜ": "t",
+  "€": "e",
+  "₱": "b",
+  "₴": "h",
+  "₹": "r",
+  "₽": "p",
+  "₿": "z",
+  "∏": "π",
+  "∑": "σ",
+  "␢": "b",
+  "Ᵽ": "p",
+  "ⱥ": "a",
+  "ⱦ": "t",
+  "ⱻ": "ǝ",
+  "ⱼ": "j",
+  "Ꙁ": "з",
+  "ꙁ": "з",
+  "Ꙉ": "ђ",
+  "ꙉ": "ђ",
+  "Ꙋ": "у",
+  "ꙋ": "у",
+  "Ꙍ": "ѡ",
+  "ꙍ": "ѡ",
+  "Ꙑ": "ы",
+  "ꙑ": "ы",
+  "Ꙗ": "я",
+  "ꙗ": "я",
+  "ꜰ": "f",
+  "ꜱ": "s",
+  "Ꝁ": "k",
+  "ꝁ": "k",
+  "Ꝃ": "k",
+  "ꝃ": "k",
+  "Ꝗ": "q",
+  "ꝗ": "q",
+  "Ꝟ": "v",
+  "ꝟ": "v",
+  "ꝶ": "ꝵ",
+  "Ꞓ": "c",
+  "ꞓ": "c",
+  "Ꞙ": "f",
+  "ꞙ": "f",
+  "Ꞡ": "g",
+  "ꞡ": "g",
+  "Ꞥ": "n",
+  "ꞥ": "n",
+  "Ꞧ": "r",
+  "ꞧ": "r",
+  "Ꞩ": "s",
+  "ꞩ": "s",
+  "Ɪ": "i",
+  "ꞯ": "q",
+  "Ꞹ": "u",
+  "ꞹ": "u",
+  "ꟴ": "q",
+  "ꟹ": "o",
+  "ꟺ": "ɯ",
+  "ꬿ": "ɔ",
+  "ꭥ": "ω",
+  "ꭩ": "ʍ"
+}

srcs/juloo.keyboard2/ComposeKey.java

@@ -22,14 +22,14 @@ public final class ComposeKey
     return null;
   }
 
-  /** Apply the pending compose sequence to char [c]. Returns [null] if no
+  /** Apply a char to the pending compose sequence. Returns [null] if no
       sequence matched. */
-  public static KeyValue apply(int prev, char c)
+  public static KeyValue apply(int state, char c)
   {
     char[] states = ComposeKeyData.states;
     char[] edges = ComposeKeyData.edges;
-    int prev_length = edges[prev];
-    int next = Arrays.binarySearch(states, prev + 1, prev + prev_length, c);
+    int state_length = edges[state];
+    int next = Arrays.binarySearch(states, state + 1, state + state_length, c);
     if (next < 0)
       return null;
     next = edges[next];
@@ -46,6 +46,25 @@ public final class ComposeKey
       return KeyValue.makeCharKey((char)next_header);
   }
 
+  /** Apply char [c] to the pending compose sequence. If the application resolves
+      to a final char state, return it, otherwise return [0]. This is faster
+      than [apply] but do not support sequences of more than 1 char. Used to
+      apply substitutions for word suggestions. */
+  public static char transform_char(int state, char c)
+  {
+    char[] states = ComposeKeyData.states;
+    char[] edges = ComposeKeyData.edges;
+    int state_length = edges[state];
+    int next = Arrays.binarySearch(states, state + 1, state + state_length, c);
+    if (next < 0)
+      return 0;
+    next = edges[next];
+    int next_header = states[next];
+    if (next_header == 0 || next_header == 0xFFFF)
+      return 0;
+    return (char)next_header;
+  }
+
   /** Apply each char of a string to a sequence. Returns [null] if no sequence
       matched. */
   public static KeyValue apply(int prev, String s)

srcs/juloo.keyboard2/ComposeKeyData.java

@@ -127,7 +127,22 @@ public final class ComposeKeyData
     "\u0bee\u0bef\u0000\u00df\u0131\u01f0\u0237\u02b0\u02b2\u02b3\u02b7\u02e1\u0905\u0907\u0909\u090b\u090c\u090f\u0913\u0915\u0917\u091a\u091c\u091f\u0921\u0924\u0926\u0928\u092c\u092e\u0932\u0938\u0939\u093f\u0941\u0943\u0945\u0947\u0949\u094b\u0952\u0962\u0a85\u0a87\u0a89\u0a8f\u0a93\u0a95\u0a97\u0a9a\u0a9c\u0a9f\u0aa1\u0aa4\u0aa6\u0aa8\u0aaa\u0aac\u0aae\u0ab2\u0ab8\u0ab9\u0abf\u0ac1\u0ac7\u0acb\u0bf9\u1d43\u1d47\u1d48\u1d49\u1d4d" +
     "\u1d4f\u1d50\u1d52\u1d56\u1d57\u1d58\u1d5b\u1d60\u1d9c\u1da0\u1dbe\u1e97\u1e98\u1e99\u2071\u207f\u20b9\u2190\u2191\u2192\u2193\u2196\u2197\u2198\u2199\u2208\u220b\u2282\u2283\u2286\u2287\u2500\u2502\u250c\u2510\u2514\u2518\u251c\u2524\u252c\u2534\u253c\ud835\uFFFF\u004a\u030c\uFFFF\u004a\u0307\u1d34\u1d36\u1d3f\u1d42\u1d38\u0906\u0908\u090a\u0910\u0914\u0916\u0918\u091b\u091d\u0920\u0922\u0925\u0927\u0923\u092d\u0902\u0933\u0936" +
     "\u0903\u0940\u0942\u0948\u094c\u0951\u0a86\u0a88\u0a8a\u0a90\u0a94\u0a96\u0a98\u0a9b\u0a9d\u0aa0\u0aa2\u0aa5\u0aa7\u0aa3\u0aab\u0aad\u0a82\u0ab3\u0ab6\u0a83\u0ac0\u0ac2\u0ac8\u0acc\u1d2c\u1d2e\u1d30\u1d31\u1d33\u1d37\u1d39\u1d3c\u1d3e\u1d40\u1d41\u2c7d\u1db2\uFFFF\ua7f2\uFFFF\ua7f3\uFFFF\u0054\u0308\uFFFF\u0057\u030a\uFFFF\u0059\u030a\u1d35\u1d3a\u2550\u2551\u2554\u2557\u255a\u255d\u2560\u2563\u2566\u2569\u256c\u0000\udd57\udd58" +
-    "\udd64\udd68\udd69\uFFFF\ud835\udd3d\uFFFF\ud835\udd3e\uFFFF\ud835\udd4a\uFFFF\ud835\udd4e\uFFFF\ud835\udd4f").toCharArray();
+    "\udd64\udd68\udd69\uFFFF\ud835\udd3d\uFFFF\ud835\udd3e\uFFFF\ud835\udd4a\uFFFF\ud835\udd4e\uFFFF\ud835\udd4f\u0000ABCDEFGHIJKLMNOPQRSTUVWXYZx\u00a2\u00a3\u00a5\u00aa\u00b5\u00ba\u00c0\u00c1\u00c2\u00c3\u00c4\u00c5\u00c6\u00c7\u00c8\u00c9\u00ca\u00cb\u00cc\u00cd\u00ce\u00cf\u00d1\u00d2\u00d3\u00d4" +
+    "\u00d5\u00d6\u00d8\u00d9\u00da\u00db\u00dc\u00dd\u00e0\u00e1\u00e2\u00e3\u00e4\u00e5\u00e6\u00e7\u00e8\u00e9\u00ea\u00eb\u00ec\u00ed\u00ee\u00ef\u00f1\u00f2\u00f3\u00f4\u00f5\u00f6\u00f8\u00f9\u00fa\u00fb\u00fc\u00fd\u00ff\u0100\u0101\u0104\u0105\u0106\u0107\u0108\u0109\u010a\u010b\u010c\u010d\u010e\u010f\u0110\u0111\u0112\u0113\u0116\u0117\u0118\u0119\u011a\u011b\u011c\u011d\u0120\u0121\u0122\u0123\u0124\u0125\u0126\u0127\u0128" +
+    "\u0129\u012a\u012b\u012e\u012f\u0130\u0131\u0134\u0135\u0136\u0137\u0139\u013a\u013b\u013c\u013d\u013e\u0141\u0142\u0143\u0144\u0145\u0146\u0147\u0148\u014c\u014d\u0150\u0151\u0152\u0153\u0154\u0155\u0156\u0157\u0158\u0159\u015a\u015b\u015c\u015d\u015e\u015f\u0160\u0161\u0162\u0163\u0164\u0165\u0166\u0167\u0168\u0169\u016a\u016b\u016e\u016f\u0170\u0171\u0172\u0173\u0174\u0175\u0176\u0177\u0178\u0179\u017a\u017b\u017c\u017d\u017e" +
+    "\u0180\u0194\u0197\u019a\u019f\u01a0\u01a1\u01a6\u01af\u01b0\u01b5\u01b6\u01cd\u01ce\u01cf\u01d0\u01d1\u01d2\u01d3\u01d4\u01d5\u01d6\u01d7\u01d8\u01d9\u01da\u01db\u01dc\u01de\u01df\u01e0\u01e1\u01e2\u01e3\u01e4\u01e5\u01e6\u01e7\u01e8\u01e9\u01ea\u01eb\u01ec\u01ed\u01ee\u01ef\u01f0\u01f4\u01f5\u01f8\u01f9\u01fa\u01fb\u01fc\u01fd\u01fe\u01ff\u0200\u0201\u0204\u0205\u0208\u0209\u020c\u020d\u0210\u0211\u0214\u0215\u021e\u021f\u0226" +
+    "\u0227\u0228\u0229\u022a\u022b\u022c\u022d\u022e\u022f\u0230\u0231\u0232\u0233\u0237\u023a\u023b\u023c\u023d\u023e\u0243\u0244\u0246\u0247\u0248\u0249\u024c\u024d\u024e\u024f\u025f\u0262\u0263\u0268\u026a\u0274\u0275\u0276\u0280\u0289\u028f\u0299\u029b\u029c\u029f\u02b0\u02b1\u02b2\u02b3\u02b4\u02b5\u02b6\u02b7\u02b8\u02e0\u02e1\u02e2\u02e3\u0386\u0388\u0389\u038a\u038c\u038e\u0392\u0395\u0398\u039a\u039c\u03a0\u03a1\u03a3\u03a6" +
+    "\u03aa\u03ab\u03ac\u03ad\u03ae\u03af\u03b0\u03c2\u03ca\u03cb\u03cc\u03cd\u03d0\u03d1\u03d2\u03d4\u03d5\u03d6\u03f0\u03f1\u03f4\u03f5\u0400\u0402\u0403\u0404\u0405\u0406\u0408\u040b\u040c\u040d\u040f\u0418\u0419\u0427\u042a\u042b\u042c\u042f\u0438\u0439\u0447\u044a\u044b\u044c\u044f\u0450\u0452\u0453\u0454\u0455\u0456\u0458\u045b\u045c\u045d\u045f\u0462\u0463\u0466\u0467\u046a\u046b\u0470\u0471\u0472\u0473\u0476\u0477\u047a\u047b" +
+    "\u0480\u0481\u048c\u048d\u0490\u0491\u0498\u0499\u049a\u049b\u04b0\u04b1\u04b6\u04b7\u04ba\u04bb\u04d2\u04d3\u04da\u04db\u04dc\u04dd\u04de\u04df\u04e2\u04e3\u04e4\u04e5\u04e6\u04e7\u04e8\u04e9\u04ea\u04eb\u04ec\u04ed\u04ee\u04ef\u04f0\u04f1\u04f2\u04f3\u04f4\u04f5\u04f8\u04f9\u04fe\u04ff\u0528\u0529\u052e\u052f\u0621\u0622\u0630\u0635\u0636\u0637\u0638\u063a\u0640\u0643\u0647\u064a\u066e\u067e\u0695\u0698\u06a1\u06a4\u06a9\u06af" +
+    "\u06b5\u06ba\u06be\u06c1\u06cc\u0902\u0903\u0906\u0908\u090a\u090d\u090e\u0910\u0911\u0912\u0914\u0916\u0918\u091b\u091d\u0920\u0922\u0923\u0925\u0927\u0929\u092d\u0931\u0933\u0934\u0936\u0958\u0959\u095a\u095b\u095c\u095d\u095e\u095f\u0960\u0961\u0972\u0973\u0974\u0975\u0976\u0977\u0978\u0979\u097a\u097b\u097c\u097d\u097e\u097f\u0a82\u0a83\u0a86\u0a88\u0a8a\u0a90\u0a94\u0a96\u0a98\u0a9b\u0a9d\u0aa0\u0aa2\u0aa3\u0aa5\u0aa7\u0aab" +
+    "\u0aad\u0ab3\u0ab6\u1d00\u1d01\u1d03\u1d04\u1d05\u1d06\u1d07\u1d0a\u1d0b\u1d0c\u1d0d\u1d0e\u1d0f\u1d10\u1d15\u1d18\u1d19\u1d1a\u1d1b\u1d1c\u1d20\u1d21\u1d22\u1d23\u1d26\u1d27\u1d28\u1d29\u1d2a\u1d2b\u1d43\u1d44\u1d46\u1d47\u1d48\u1d49\u1d4a\u1d4b\u1d4c\u1d4d\u1d4e\u1d4f\u1d50\u1d51\u1d52\u1d53\u1d56\u1d57\u1d58\u1d59\u1d5a\u1d5b\u1d5d\u1d5f\u1d60\u1d61\u1d62\u1d63\u1d64\u1d65\u1d66\u1d67\u1d68\u1d69\u1d6a\u1d7b\u1d7d\u1d7e\u1d9b" +
+    "\u1d9c\u1d9d\u1d9f\u1da0\u1da1\u1da3\u1da4\u1da5\u1dac\u1dad\u1db1\u1db3\u1db4\u1db6\u1db7\u1dba\u1dbb\u1dbe\u1dbf\u1e02\u1e03\u1e04\u1e05\u1e08\u1e09\u1e0a\u1e0b\u1e0c\u1e0d\u1e10\u1e11\u1e14\u1e15\u1e16\u1e17\u1e1c\u1e1d\u1e1e\u1e1f\u1e20\u1e21\u1e22\u1e23\u1e24\u1e25\u1e26\u1e27\u1e28\u1e29\u1e2e\u1e2f\u1e30\u1e31\u1e32\u1e33\u1e36\u1e37\u1e38\u1e39\u1e3e\u1e3f\u1e40\u1e41\u1e42\u1e43\u1e44\u1e45\u1e46\u1e47\u1e4c\u1e4d\u1e4e" +
+    "\u1e4f\u1e50\u1e51\u1e52\u1e53\u1e54\u1e55\u1e56\u1e57\u1e58\u1e59\u1e5a\u1e5b\u1e5c\u1e5d\u1e60\u1e61\u1e62\u1e63\u1e64\u1e65\u1e66\u1e67\u1e68\u1e69\u1e6a\u1e6b\u1e6c\u1e6d\u1e78\u1e79\u1e7c\u1e7d\u1e7e\u1e7f\u1e80\u1e81\u1e82\u1e83\u1e84\u1e85\u1e86\u1e87\u1e88\u1e89\u1e8a\u1e8b\u1e8c\u1e8d\u1e8e\u1e8f\u1e90\u1e91\u1e92\u1e93\u1e97\u1e98\u1e99\u1e9b\u1e9e\u1ea0\u1ea1\u1ea2\u1ea3\u1ea4\u1ea5\u1ea6\u1ea7\u1ea8\u1ea9\u1eaa\u1eab" +
+    "\u1eac\u1ead\u1eae\u1eaf\u1eb0\u1eb1\u1eb2\u1eb3\u1eb4\u1eb5\u1eb6\u1eb7\u1eb8\u1eb9\u1eba\u1ebb\u1ebc\u1ebd\u1ebe\u1ebf\u1ec0\u1ec1\u1ec2\u1ec3\u1ec4\u1ec5\u1ec6\u1ec7\u1ec8\u1ec9\u1eca\u1ecb\u1ecc\u1ecd\u1ece\u1ecf\u1ed0\u1ed1\u1ed2\u1ed3\u1ed4\u1ed5\u1ed6\u1ed7\u1ed8\u1ed9\u1eda\u1edb\u1edc\u1edd\u1ede\u1edf\u1ee0\u1ee1\u1ee2\u1ee3\u1ee4\u1ee5\u1ee6\u1ee7\u1ee8\u1ee9\u1eea\u1eeb\u1eec\u1eed\u1eee\u1eef\u1ef0\u1ef1\u1ef2\u1ef3" +
+    "\u1ef4\u1ef5\u1ef6\u1ef7\u1ef8\u1ef9\u1f70\u1f72\u1f74\u1f76\u1f78\u1f7a\u1f7c\u1fb1\u1fb9\u1fba\u1fc8\u1fca\u1fd1\u1fd9\u1fda\u1fe1\u1fe2\u1fe7\u1fe9\u1fea\u1ff8\u1ffa\u2071\u207f\u2090\u2091\u2092\u2093\u2094\u2095\u2096\u2097\u2098\u2099\u209a\u209b\u209c\u20ac\u20b1\u20b4\u20b9\u20bd\u20bf\u220f\u2211\u2422\u2c63\u2c65\u2c66\u2c7b\u2c7c\ua640\ua641\ua648\ua649\ua64a\ua64b\ua64c\ua64d\ua650\ua651\ua656\ua657\ua730\ua731\ua740" +
+    "\ua741\ua742\ua743\ua756\ua757\ua75e\ua75f\ua776\ua792\ua793\ua798\ua799\ua7a0\ua7a1\ua7a4\ua7a5\ua7a6\ua7a7\ua7a8\ua7a9\ua7ae\ua7af\ua7b8\ua7b9\ua7f4\ua7f9\ua7fa\uab3f\uab65\uab69abcdefghijklmnopqrstuvwyz\u03b3\u0292\u0260\u0266\u0279\u027b\u0281\u03b1\u03b5\u03b7\u03b9\u03bf\u03c5\u03b2\u03b8\u03ba\u03c0" +
+    "\u03c1\u03c3\u03c6\u0435\u0433\u044d\u0441\u043a\u043d\u043c\u0448\u0444\u043e\u0434\u04af\u0445\u0430\u0436\u0437\u0443\u043b\u0639\u0627\u062f\u0633\u062a\u0632\u0642\u0645\u0628\u0631\u0648\u0641\u0644\u0646\u062d\u092e\u0939\u0905\u0907\u0909\u090f\u0913\u0915\u0917\u091a\u091c\u091f\u0921\u0928\u0924\u0926\u092c\u0930\u0932\u0938\u092b\u092f\u090b\u090c\u0935\u093d\u0aae\u0ab9\u0a85\u0a87\u0a89\u0a8f\u0a93\u0a95\u0a97\u0a9a" +
+    "\u0a9c\u0a9f\u0aa1\u0aa8\u0aa4\u0aa6\u0aaa\u0aac\u0ab2\u0ab8\u0254\u0223\u03bb\u03c8\u0250\u1d02\u025b\u1d08\u1d09\u1d1d\u026f\u03b4\u03c7\u0252\u0255\u025c\u0265\u0271\u0270\u0282\u0283\u028a\u028c\u03c9\u01dd\uFFFF\ua775\u028d").toCharArray();
 
   public static final char[] edges =
     ("\u0001\u0036\u0037\u0038\u0039\u003a\u003b\u003c\u003f\u0040\u0041\u0042\u0043\u0044\u0045\u0046\u0047\u0048\u0049\u004a\u004b\u004c\u004d\u004e\u004f\u0050\u0051\u0052\u0053\u0054\u0055\u0056\u0059\u005a\u005b\\\u005d\u005e\u005f\u0060\u0061\u0062\u0063\u0064\u0067\u0068\u006b\u006e\u006f\u0072\u0075\u0078\u007b\u007e\u0081\u0001\u0001\u0001\u0001\u0001\u0003\u0000\u0000\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001" +
@@ -252,7 +267,22 @@ public final class ComposeKeyData
     "\u0001\u0001\u0071\u1016\u0daf\u21eb\u21ee\u21f1\u21f2\u21f3\u21f4\u21f5\u21f6\u21f7\u21f8\u2081\u2082\u21f9\u21fa\u21fb\u21fc\u21fd\u21fe\u21ff\u2200\u2201\u2202\u2203\u2204\u2205\u2206\u2207\u2208\u2209\u220a\u20ac\u207b\u220b\u207c\u220c\u220d\u20b3\u220e\u220f\u2210\u2211\u2212\u2213\u2214\u2215\u2216\u2217\u2218\u2219\u221a\u221b\u221c\u221d\u221e\u221f\u2220\u2221\u2222\u2223\u2224\u2225\u1008\u2226\u2227\u2228\u2229\u222a" +
     "\u222b\u222c\u222d\u222e\u222f\u2230\u2231\u2232\u2233\u2235\u03f2\u2237\u223a\u223d\u2240\u2241\u1008\u0eb9\u0ec5\u0eba\u0ec8\u20c2\u20c3\u20c4\u20c5\u1dd7\u1ddc\u1e32\u1e36\u1e39\u1e3c\u2242\u2243\u2244\u2245\u2246\u2247\u2248\u2249\u224a\u224b\u224c\u224d\u0003\u0000\u0000\u0003\u0000\u0000\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001" +
     "\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0002\u0000\u0002\u0000\u0003\u0000\u0000\u0003\u0000\u0000\u0003\u0000\u0000\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0006\u2253\u2256" +
-    "\u2259\u225c\u225f\u0003\u0000\u0000\u0003\u0000\u0000\u0003\u0000\u0000\u0003\u0000\u0000\u0003\u0000\u0000").toCharArray();
+    "\u2259\u225c\u225f\u0003\u0000\u0000\u0003\u0000\u0000\u0003\u0000\u0000\u0003\u0000\u0000\u0003\u0000\u0000\u03b4\u2616\u2617\u2618\u2619\u261a\u261b\u261c\u261d\u261e\u261f\u2620\u2621\u2622\u2623\u2624\u2625\u2626\u2627\u2628\u2629\u262a\u262b\u262c\u03e3\u262d\u262e\u03e3\u2618\u2621\u262d\u2616\u262a\u2624\u2616\u2616\u2616\u2616\u2616\u2616\u2616\u2618\u261a\u261a\u261a\u261a\u261e\u261e\u261e\u261e\u2623\u2624\u2624\u2624" +
+    "\u2624\u2624\u2624\u262a\u262a\u262a\u262a\u262d\u2616\u2616\u2616\u2616\u2616\u2616\u2616\u2618\u261a\u261a\u261a\u261a\u261e\u261e\u261e\u261e\u2623\u2624\u2624\u2624\u2624\u2624\u2624\u262a\u262a\u262a\u262a\u262d\u262d\u2616\u2616\u2616\u2616\u2618\u2618\u2618\u2618\u2618\u2618\u2618\u2618\u2619\u2619\u2619\u2619\u261a\u261a\u261a\u261a\u261a\u261a\u261a\u261a\u261c\u261c\u261c\u261c\u261c\u261c\u261d\u261d\u261d\u261d\u261e" +
+    "\u261e\u261e\u261e\u261e\u261e\u261e\u261e\u261f\u261f\u2620\u2620\u2621\u2621\u2621\u2621\u2621\u2621\u2621\u2621\u2623\u2623\u2623\u2623\u2623\u2623\u2624\u2624\u2624\u2624\u2624\u2624\u2627\u2627\u2627\u2627\u2627\u2627\u2628\u2628\u0184\u0184\u2628\u2628\u2628\u2628\u2629\u2629\u2629\u2629\u2629\u2629\u262a\u262a\u262a\u262a\u262a\u262a\u262a\u262a\u262a\u262a\u262c\u262c\u262d\u262d\u262d\u262e\u262e\u262e\u262e\u262e\u262e" +
+    "\u2617\u262f\u261e\u2621\u2624\u2624\u2624\u2627\u262a\u262a\u262e\u262e\u2616\u2616\u261e\u261e\u2624\u2624\u262a\u262a\u262a\u262a\u262a\u262a\u262a\u262a\u262a\u262a\u2616\u2616\u2616\u2616\u2616\u2616\u261c\u261c\u261c\u261c\u2620\u2620\u2624\u2624\u2624\u2624\u2630\u2630\u261f\u261c\u261c\u2623\u2623\u2616\u2616\u2616\u2616\u2624\u2624\u2616\u2616\u261a\u261a\u261e\u261e\u2624\u2624\u2627\u2627\u262a\u262a\u261d\u261d\u2616" +
+    "\u2616\u261a\u261a\u2624\u2624\u2624\u2624\u2624\u2624\u2624\u2624\u262d\u262d\u261f\u2616\u2618\u2618\u2621\u2629\u2617\u262a\u261a\u261a\u261f\u261f\u2627\u2627\u262d\u262d\u261f\u261c\u262f\u261e\u261e\u2623\u2624\u2624\u2627\u262a\u262d\u2617\u2631\u261d\u2621\u261d\u2632\u261f\u2627\u2633\u2634\u2635\u262c\u262d\u262f\u2621\u2628\u03e3\u2636\u2637\u2638\u2639\u263a\u263b\u263c\u2637\u263d\u263e\u262a\u263f\u2640\u2641\u2642" +
+    "\u2639\u263b\u2636\u2637\u2638\u2639\u263b\u2641\u2639\u263b\u263a\u263b\u263c\u263d\u263b\u263b\u2642\u263f\u263e\u2640\u263d\u2637\u2643\u17f9\u2644\u2645\u2646\u1fea\u107c\u1feb\u2647\u1fea\u0dce\u1fdc\u1fdd\u187c\u1fe5\u1fe6\u1fe8\u191f\u1fdc\u1fdd\u187c\u1fe5\u1fe6\u1fe8\u191f\u2643\u17f9\u2644\u2645\u2646\u1fea\u107c\u1feb\u2647\u1fea\u0dce\u2643\u2643\u2648\u2648\u2649\u2649\u264a\u264a\u264b\u264b\u181a\u181a\u264c\u264c" +
+    "\u0dca\u0dca\u1fe5\u1fe5\u2644\u2644\u264d\u264d\u17f1\u17f1\u264e\u264e\u17fb\u17fb\u264f\u264f\u2650\u2650\u17cb\u17cb\u2651\u2651\u2652\u2652\u1fea\u1fea\u1fea\u1fea\u264c\u264c\u264c\u264c\u264c\u264c\u2645\u2645\u2653\u2653\u2653\u2653\u2653\u2653\u1feb\u1feb\u20cc\u20cc\u264f\u264f\u0d05\u0d05\u2654\u2654\u2655\u2656\u2657\u2658\u2658\u2659\u265a\u265b\u265c\u2071\u2074\u2075\u265d\u19e8\u265e\u265a\u265f\u2660\u2076\u265b" +
+    "\u2661\u2662\u2077\u2663\u2078\u2664\u2665\u2666\u2667\u2668\u2669\u2669\u2669\u2666\u266a\u266a\u266b\u266c\u266d\u266e\u266f\u2670\u2671\u2672\u2673\u2671\u2674\u2675\u2676\u2676\u2677\u266b\u266b\u266c\u266e\u2670\u2670\u2678\u2679\u267a\u267b\u2666\u2667\u2667\u266a\u2668\u2668\u266f\u266e\u267c\u266c\u266d\u267d\u2671\u2674\u267e\u267f\u2680\u2681\u2682\u2683\u2684\u2685\u2686\u2687\u2688\u2689\u268a\u268b\u268c\u268d\u268e" +
+    "\u268f\u2690\u2691\u2616\u2616\u2617\u2618\u2619\u1357\u261a\u261f\u2620\u2621\u2622\u13b7\u2624\u2692\u2693\u2625\u20cd\u2633\u2629\u262a\u262b\u262c\u262e\u2630\u262f\u2694\u263f\u2640\u2695\u2654\u2616\u2696\u2697\u2617\u2619\u261a\u136a\u2698\u2699\u261c\u269a\u2620\u2622\u13b7\u2624\u2692\u2625\u2629\u262a\u269b\u269c\u262b\u263c\u269d\u2642\u269e\u261e\u2627\u262a\u262b\u263c\u262f\u2640\u2642\u269e\u261e\u2625\u262a\u269f" +
+    "\u2618\u26a0\u26a1\u261b\u261f\u26a2\u261e\u2639\u26a3\u26a4\u2624\u26a5\u26a6\u262a\u26a7\u26a8\u262e\u2630\u263d\u2617\u2617\u2617\u2617\u2618\u2618\u2619\u2619\u2619\u2619\u2619\u2619\u261a\u261a\u261a\u261a\u1065\u1065\u261b\u261b\u261c\u261c\u261d\u261d\u261d\u261d\u261d\u261d\u261d\u261d\u261e\u261e\u2620\u2620\u2620\u2620\u2621\u2621\u2621\u2621\u2622\u2622\u2622\u2622\u2622\u2622\u2623\u2623\u2623\u2623\u2624\u2624\u2624" +
+    "\u2624\u2624\u2624\u2624\u2624\u2625\u2625\u2625\u2625\u2627\u2627\u2627\u2627\u2627\u2627\u2628\u2628\u2628\u2628\u2628\u2628\u2628\u2628\u2628\u2628\u2629\u2629\u2629\u2629\u262a\u262a\u262b\u262b\u262b\u262b\u262c\u262c\u262c\u262c\u262c\u262c\u262c\u262c\u262c\u262c\u03e3\u03e3\u03e3\u03e3\u262d\u262d\u262e\u262e\u262e\u262e\u2629\u262c\u262d\u1372\u13ed\u2616\u2616\u2616\u2616\u2616\u2616\u2616\u2616\u2616\u2616\u2616\u2616" +
+    "\u2616\u2616\u1064\u1064\u1064\u1064\u1064\u1064\u1064\u1064\u1064\u1064\u261a\u261a\u261a\u261a\u261a\u261a\u261a\u261a\u261a\u261a\u261a\u261a\u261a\u261a\u261a\u261a\u261e\u261e\u261e\u261e\u2624\u2624\u2624\u2624\u2624\u2624\u2624\u2624\u2624\u2624\u2624\u2624\u2624\u2624\u2624\u2624\u2624\u2624\u2624\u2624\u2624\u2624\u2624\u2624\u262a\u262a\u262a\u262a\u262a\u262a\u262a\u262a\u262a\u262a\u262a\u262a\u262a\u262a\u262d\u262d" +
+    "\u262d\u262d\u262d\u262d\u262d\u262d\u2636\u2637\u2638\u2639\u263a\u263b\u26a9\u2636\u2636\u2636\u2637\u2638\u2639\u2639\u2639\u263b\u263b\u14df\u263b\u263b\u263a\u26a9\u261e\u2623\u2616\u261a\u2624\u03e3\u136a\u261d\u2620\u2621\u2622\u2623\u2625\u2628\u2629\u261a\u2617\u261d\u2627\u2625\u262e\u263f\u2641\u2617\u2625\u2616\u2629\u26aa\u261f\u2652\u2652\u17fe\u17fe\u2653\u2653\u17b8\u17b8\u20cc\u20cc\u20cd\u20cd\u261b\u2628\u2620" +
+    "\u2620\u2620\u2620\u2626\u2626\u262b\u262b\u26ab\u2618\u2618\u261b\u261b\u261c\u261c\u2623\u2623\u2627\u2627\u2628\u2628\u261e\u2626\u262a\u262a\u2626\u2624\u269c\u2692\u26a9\u26ad\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001" +
+    "\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001" +
+    "\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0002\u0000\u0001").toCharArray();
 
   public static final int accent_aigu = 1;
   public static final int accent_arrows = 130;
@@ -288,4 +318,5 @@ public final class ComposeKeyData
   public static final int numpad_persian = 8528;
   public static final int numpad_tamil = 8549;
   public static final int shift = 8570;
+  public static final int substitutions = 8802;
 }

srcs/juloo.keyboard2/dict/DictionaryListView.java

@@ -155,7 +155,7 @@ public class DictionaryListView extends LinearLayout
   static URL url_of_dictionary(String dict_name)
       throws MalformedURLException
   {
-    int format_version = 0;
+    int format_version = Cdict.format_version();
     return new URL(DICT_REPO_URL + "/v" + format_version + "/" + dict_name
         + ".dict");
   }

srcs/juloo.keyboard2/suggestions/Suggestions.java

@@ -5,6 +5,8 @@ import java.util.List;
 import juloo.cdict.Cdict;
 import juloo.keyboard2.dict.Dictionaries;
 import juloo.keyboard2.Config;
+import juloo.keyboard2.ComposeKey;
+import juloo.keyboard2.ComposeKeyData;
 
 /** Keep track of the word being typed and provide suggestions for
     [CandidatesView]. */
@@ -49,18 +51,12 @@ public final class Suggestions
 
   int query_suggestions(Cdict dict, String word, String[] dst, int max_count)
   {
+    boolean first_char_upper = Character.isUpperCase(word.charAt(0));
+    word = apply_substitutions(word);
     Cdict.Result r = dict.find(word);
     int i = 0;
     if (r.found)
-      dst[i++] = word;
-    boolean first_char_upper = Character.isUpperCase(word.charAt(0));
-    // Do the dictionary query in lower case and re-apply the upper case after
-    if (first_char_upper)
-    {
-      r = dict.find(word.toLowerCase());
-      if (r.found)
-        dst[i++] = word;
-    }
+      dst[i++] = dict.word(r.index);
     int[] suffixes = dict.suffixes(r, max_count);
     // Disable distance search for small words
     int[] dist = (word.length() < 3 || i + 1 >= max_count) ? NO_RESULTS :
@@ -84,6 +80,22 @@ public final class Suggestions
         rs[i] = rs[i].substring(0, 1).toUpperCase() + rs[i].substring(1);
   }
 
+  /** Apply the same substitutions that were used when building the
+      dictionaries to find word aliases. This catches missing diacritics for
+      example. */
+  String apply_substitutions(String w)
+  {
+    StringBuilder b = new StringBuilder(w);
+    int len = w.length();
+    for (int i = 0; i < len; i++)
+    {
+      char r =
+        ComposeKey.transform_char(ComposeKeyData.substitutions, b.charAt(i));
+      if (r != 0) b.setCharAt(i, r);
+    }
+    return b.toString();
+  }
+
   void set_suggestions(List<String> ws)
   {
     _callback.set_suggestions(ws);