From aa4fea155b2eb01d8e18e5cb104e43e7b9daec17 Mon Sep 17 00:00:00 2001 From: Nick White Date: Mon, 20 Nov 2017 14:18:39 +0000 Subject: [PATCH] Handle combining characters correctly in a codec Using combining characters in a codec would result in the combining codepoint(s) being separated from their base character. Now combining characters will be included with their base character, so the whole grapheme will be considered as one. This means that complex characters, which are made up of multiple unicode codepoints, can now be correctly used and represented with Ocropus. Note that I don't think this covers all possible Unicode "grapheme clusters" as defined here: http://unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries It only works with combining characters. But that still satisfies a lot of use cases. --- ocrolib/lstm.py | 14 +++++++++++++- ocropus-rtrain | 5 +++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/ocrolib/lstm.py b/ocrolib/lstm.py index bf10ab56..8015e4ab 100644 --- a/ocrolib/lstm.py +++ b/ocrolib/lstm.py @@ -102,6 +102,18 @@ def sumprod(us,vs,lo=-1.0,hi=1.0,out=None): result += clip(u,lo,hi)*v return result +def graphemelist(str): + l = list() + s = "" + for char in str: + if unicodedata.category(char)[0] == 'M': + s += char + else: + l.append(s) + s = char + l.append(s) + return l + class Network: """General interface for networks. This mainly adds convenience functions for `predict` and `train`. @@ -963,7 +975,7 @@ def encode(self,s): "Encode the string `s` into a code sequence." # tab = self.char2code dflt = self.char2code["~"] - return [self.char2code.get(c,dflt) for c in s] + return [self.char2code.get(c,dflt) for c in graphemelist(s)] def decode(self,l): "Decode a code sequence into a string." s = [self.code2char.get(c,"~") for c in l] diff --git a/ocropus-rtrain b/ocropus-rtrain index 9d365ae3..abfb7da8 100755 --- a/ocropus-rtrain +++ b/ocropus-rtrain @@ -134,13 +134,14 @@ if args.codec!=[]: print(args.codec) for fname in ocrolib.glob_all(args.codec): transcript = ocrolib.read_text(fname) - l = list(lstm.normalize_nfkc(transcript)) + l = lstm.graphemelist(lstm.normalize_nfkc(transcript)) charset = charset.union(l) charset = sorted(list(charset)) charset = [c for c in charset if c>" " and c!="~"] else: print("# using default codec") - charset = sorted(list(set(list(lstm.ascii_labels) + list(ocrolib.chars.default)))) + chars = "".join(lstm.ascii_labels) + ocrolib.chars.default + charset = sorted(list(set(lstm.graphemelist(chars)))) charset = [""," ","~",]+[c for c in charset if c not in [" ","~"]] print("# charset size", len(charset), end=' ')