Handle combining characters correctly in a codec

ocrolib/lstm.py

@@ -102,6 +102,18 @@ def sumprod(us,vs,lo=-1.0,hi=1.0,out=None):
         result += clip(u,lo,hi)*v
     return result
 
+def graphemelist(str):
+    l = list()
+    s = ""
+    for char in str:
+        if unicodedata.category(char)[0] == 'M':
+            s += char
+        else:
+            l.append(s)
+            s = char
+    l.append(s)
+    return l
+
 class Network:
     """General interface for networks. This mainly adds convenience
     functions for `predict` and `train`.
@@ -963,7 +975,7 @@ def encode(self,s):
         "Encode the string `s` into a code sequence."
         # tab = self.char2code
         dflt = self.char2code["~"]
-        return [self.char2code.get(c,dflt) for c in s]
+        return [self.char2code.get(c,dflt) for c in graphemelist(s)]
     def decode(self,l):
         "Decode a code sequence into a string."
         s = [self.code2char.get(c,"~") for c in l]

ocropus-rtrain

@@ -134,13 +134,14 @@ if args.codec!=[]:
     print(args.codec)
     for fname in ocrolib.glob_all(args.codec):
         transcript = ocrolib.read_text(fname)
-        l = list(lstm.normalize_nfkc(transcript))
+        l = lstm.graphemelist(lstm.normalize_nfkc(transcript))
         charset = charset.union(l)
     charset = sorted(list(charset))
     charset = [c for c in charset if c>" " and c!="~"]
 else:
     print("# using default codec")
-    charset = sorted(list(set(list(lstm.ascii_labels) + list(ocrolib.chars.default))))
+    chars = "".join(lstm.ascii_labels) + ocrolib.chars.default
+    charset = sorted(list(set(lstm.graphemelist(chars))))
 
 charset = [""," ","~",]+[c for c in charset if c not in [" ","~"]]
 print("# charset size", len(charset), end=' ')