Another Python2 to Python3 migration

OLD/ocrolib/ngraphs.py

@@ -88,7 +88,7 @@ def computeNGraphs(self,fnames,n):
                 continue
             with codecs.open(fname,"r","utf-8") as stream:
                 for lineno,line in enumerate(safe_readlines(stream)):
-                    assert type(line)==unicode
+                    assert isinstance(line, str)
                     if lineno<lineskip: continue
                     if lineno>=linelimit+lineskip: break
                     line = line[:-1]

ocrolib/__init__.py

@@ -10,6 +10,6 @@
 ### top level imports
 ################################################################
 
-import default
-from common import *
-from default import traceback as trace
+from . import default
+from .common import *
+from .default import traceback as trace

ocrolib/chars.py

@@ -4,18 +4,18 @@
 
 # common character sets
 
-digits = u"0123456789"
-letters = u"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
-symbols = ur"""!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~"""
+digits = "0123456789"
+letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+symbols = r"""!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~"""
 ascii = digits+letters+symbols
 
-xsymbols = u"""€¢£»«›‹÷©®†‡°∙•◦‣¶§÷¡¿▪▫"""
-german = u"ÄäÖöÜüß"
-french = u"ÀàÂâÆæÇçÉéÈèÊêËëÎîÏïÔôŒœÙùÛûÜüŸÿ"
-turkish = u"ĞğŞşıſ"
-greek = u"ΑαΒβΓγΔδΕεΖζΗηΘθΙιΚκΛλΜμΝνΞξΟοΠπΡρΣσςΤτΥυΦφΧχΨψΩω"
-portuguese = u"ÁÃÌÍÒÓÕÚáãìíòóõú"
-telugu = u" ఁంఃఅఆఇఈఉఊఋఌఎఏఐఒఓఔకఖగఘఙచఛజఝఞటఠడఢణతథదధనపఫబభమయరఱలళవశషసహఽాిీుూృౄెేైొోౌ్ౘౙౠౡౢౣ౦౧౨౩౪౫౬౭౮౯"
+xsymbols = """€¢£»«›‹÷©®†‡°∙•◦‣¶§÷¡¿▪▫"""
+german = "ÄäÖöÜüß"
+french = "ÀàÂâÆæÇçÉéÈèÊêËëÎîÏïÔôŒœÙùÛûÜüŸÿ"
+turkish = "ĞğŞşıſ"
+greek = "ΑαΒβΓγΔδΕεΖζΗηΘθΙιΚκΛλΜμΝνΞξΟοΠπΡρΣσςΤτΥυΦφΧχΨψΩω"
+portuguese = "ÁÃÌÍÒÓÕÚáãìíòóõú"
+telugu = " ఁంఃఅఆఇఈఉఊఋఌఎఏఐఒఓఔకఖగఘఙచఛజఝఞటఠడఢణతథదధనపఫబభమయరఱలళవశషసహఽాిీుూృౄెేైొోౌ్ౘౙౠౡౢౣ౦౧౨౩౪౫౬౭౮౯"
 
 default = ascii+xsymbols+german+french+portuguese
 
@@ -35,53 +35,53 @@
 # there seems to be left vs right leaning, and top-heavy vs bottom-heavy
 
 replacements = [
-    (u'[_~#]',u"~"), # OCR control characters
-    (u'"',u"''"), # typewriter double quote
-    (u"`",u"'"), # grave accent
-    (u'[“”]',u"''"), # fancy quotes
-    (u"´",u"'"), # acute accent
-    (u"[‘’]",u"'"), # left single quotation mark
-    (u"[“”]",u"''"), # right double quotation mark
-    (u"“",u"''"), # German quotes
-    (u"„",u",,"), # German quotes
-    (u"…",u"..."), # ellipsis
-    (u"′",u"'"), # prime
-    (u"″",u"''"), # double prime
-    (u"‴",u"'''"), # triple prime
-    (u"〃",u"''"), # ditto mark
-    (u"µ",u"μ"), # replace micro unit with greek character
-    (u"[–—]",u"-"), # variant length hyphens
-    (u"fl",u"fl"), # expand Unicode ligatures
-    (u"fi",u"fi"),
-    (u"ff",u"ff"),
-    (u"ffi",u"ffi"),
-    (u"ffl",u"ffl"),
+    ('[_~#]',"~"), # OCR control characters
+    ('"',"''"), # typewriter double quote
+    ("`","'"), # grave accent
+    ('[“”]',"''"), # fancy quotes
+    ("´","'"), # acute accent
+    ("[‘’]","'"), # left single quotation mark
+    ("[“”]","''"), # right double quotation mark
+    ("“","''"), # German quotes
+    ("„",",,"), # German quotes
+    ("…","..."), # ellipsis
+    ("′","'"), # prime
+    ("″","''"), # double prime
+    ("‴","'''"), # triple prime
+    ("〃","''"), # ditto mark
+    ("µ","μ"), # replace micro unit with greek character
+    ("[–—]","-"), # variant length hyphens
+    ("fl","fl"), # expand Unicode ligatures
+    ("fi","fi"),
+    ("ff","ff"),
+    ("ffi","ffi"),
+    ("ffl","ffl"),
 ]
 
 def requote(s):
-    s = unicode(s)
-    s = re.sub(ur"''",u'"',s)
+    s = str(s)
+    s = re.sub(r"''",'"',s)
     return s
 
 def requote_fancy(s,germanic=0):
-    s = unicode(s)
+    s = str(s)
     if germanic:
         # germanic quoting style reverses the shapes
         # straight double quotes
-        s = re.sub(ur"\s+''",u"”",s)
-        s = re.sub(u"''\s+",u"“",s)
-        s = re.sub(ur"\s+,,",u"„",s)
+        s = re.sub(r"\s+''","”",s)
+        s = re.sub("''\s+","“",s)
+        s = re.sub(r"\s+,,","„",s)
         # straight single quotes
-        s = re.sub(ur"\s+'",u"’",s)
-        s = re.sub(ur"'\s+",u"‘",s)
-        s = re.sub(ur"\s+,",u"‚",s)
+        s = re.sub(r"\s+'","’",s)
+        s = re.sub(r"'\s+","‘",s)
+        s = re.sub(r"\s+,","‚",s)
     else:
         # straight double quotes
-        s = re.sub(ur"\s+''",u"“",s)
-        s = re.sub(ur"''\s+",u"”",s)
-        s = re.sub(ur"\s+,,",u"„",s)
+        s = re.sub(r"\s+''","“",s)
+        s = re.sub(r"''\s+","”",s)
+        s = re.sub(r"\s+,,","„",s)
         # straight single quotes
-        s = re.sub(ur"\s+'",u"‘",s)
-        s = re.sub(ur"'\s+",u"’",s)
-        s = re.sub(ur"\s+,",u"‚",s)
+        s = re.sub(r"\s+'","‘",s)
+        s = re.sub(r"'\s+","’",s)
+        s = re.sub(r"\s+,","‚",s)
     return s

ocrolib/common.py

@@ -13,7 +13,8 @@
 import unicodedata
 import inspect
 import glob
-import cPickle
+import pickle as cPickle
+import subprocess
 from ocrolib.exceptions import (BadClassLabel, BadInput, FileNotFound,
                                 OcropusException)
 
@@ -25,16 +26,16 @@
 from scipy.ndimage import morphology, measurements
 import PIL
 
-from default import getlocal
-from toplevel import (checks, ABINARY2, AINT2, AINT3, BOOL, DARKSEG, GRAYSCALE,
+from .default import getlocal
+from .toplevel import (checks, ABINARY2, AINT2, AINT3, BOOL, DARKSEG, GRAYSCALE,
                       LIGHTSEG, LINESEG, PAGESEG)
-import chars
+from . import chars
+from . import ligatures
+from . import lstm
+from . import morph
+from . import sl
 import codecs
-import ligatures
-import lstm
-import morph
 import multiprocessing
-import sl
 
 pickle_mode = 2
 
@@ -47,37 +48,36 @@ def normalize_text(s):
     """Apply standard Unicode normalizations for OCR.
     This eliminates common ambiguities and weird unicode
     characters."""
-    s = unicode(s)
     s = unicodedata.normalize('NFC',s)
-    s = re.sub(ur'\s+(?u)',' ',s)
-    s = re.sub(ur'\n(?u)','',s)
-    s = re.sub(ur'^\s+(?u)','',s)
-    s = re.sub(ur'\s+$(?u)','',s)
+    s = re.sub(r'\s+(?u)',' ',s)
+    s = re.sub(r'\n(?u)','',s)
+    s = re.sub(r'^\s+(?u)','',s)
+    s = re.sub(r'\s+$(?u)','',s)
     for m,r in chars.replacements:
-        s = re.sub(unicode(m),unicode(r),s)
+        s = re.sub(m,r,s)
     return s
 
 def project_text(s,kind="exact"):
     """Project text onto a smaller subset of characters
     for comparison."""
     s = normalize_text(s)
-    s = re.sub(ur'( *[.] *){4,}',u'....',s) # dot rows
-    s = re.sub(ur'[~_]',u'',s) # dot rows
+    s = re.sub(r'( *[.] *){4,}',u'....',s) # dot rows
+    s = re.sub(r'[~_]',u'',s) # dot rows
     if kind=="exact":
         return s
     if kind=="nospace":
-        return re.sub(ur'\s','',s)
+        return re.sub(r'\s','',s)
     if kind=="spletdig":
-        return re.sub(ur'[^A-Za-z0-9 ]','',s)
+        return re.sub(r'[^A-Za-z0-9 ]','',s)
     if kind=="letdig":
-        return re.sub(ur'[^A-Za-z0-9]','',s)
+        return re.sub(r'[^A-Za-z0-9]','',s)
     if kind=="letters":
-        return re.sub(ur'[^A-Za-z]','',s)
+        return re.sub(r'[^A-Za-z]','',s)
     if kind=="digits":
-        return re.sub(ur'[^0-9]','',s)
+        return re.sub(r'[^0-9]','',s)
     if kind=="lnc":
         s = s.upper()
-        return re.sub(ur'[^A-Z]','',s)
+        return re.sub(r'[^A-Z]','',s)
     raise BadInput("unknown normalization: "+kind)
 
 ################################################################
@@ -420,13 +420,6 @@ def save_object(fname,obj,zip=0):
         with open(fname,"wb") as stream:
             cPickle.dump(obj,stream,2)
 
-def unpickle_find_global(mname,cname):
-    if mname=="lstm.lstm":
-        return getattr(lstm,cname)
-    if not mname in sys.modules.keys():
-        exec "import "+mname
-    return getattr(sys.modules[mname],cname)
-
 def load_object(fname,zip=0,nofind=0,verbose=0):
     """Loads an object from disk. By default, this handles zipped files
     and searches in the usual places for OCRopus. It also handles some
@@ -439,15 +432,12 @@ class names that have changed."""
         zip = 1
     if zip>0:
         # with gzip.GzipFile(fname,"rb") as stream:
-        with os.popen("gunzip < '%s'"%fname,"rb") as stream:
-            unpickler = cPickle.Unpickler(stream)
-            unpickler.find_global = unpickle_find_global
-            return unpickler.load()
+        gzip = subprocess.Popen(["gzip", "-cd", fname], stdout=subprocess.PIPE)
+        with gzip.stdout as stream:
+            return cPickle.load(stream, encoding='latin1')
     else:
         with open(fname,"rb") as stream:
-            unpickler = cPickle.Unpickler(stream)
-            unpickler.find_global = unpickle_find_global
-            return unpickler.load()
+            return pickler.load(stream, encoding='latin1')
 
 
 
@@ -504,11 +494,11 @@ def parallel_map(fun,jobs,parallel=0,chunksize=1):
 def check_valid_class_label(s):
     """Determines whether the given character is a valid class label.
     Control characters and spaces are not permitted."""
-    if type(s)==unicode:
+    if type(s)==str:
         if re.search(r'[\0-\x20]',s):
             raise BadClassLabel(s)
-    elif type(s)==str:
-        if re.search(r'[^\x21-\x7e]',s):
+    elif type(s)==bytes:
+        if re.search(rb'[^\x21-\x7e]',s):
             raise BadClassLabel(s)
     else:
         raise BadClassLabel(s)
@@ -551,11 +541,11 @@ def allsplitext(path):
 def base(path):
     return allsplitext(path)[0]
 
-@checks(str,{str,unicode})
+@checks(str,str)
 def write_text_simple(file,s):
     """Write the given string s to the output file."""
     with open(file,"w") as stream:
-        if type(s)==unicode: s = s.encode("utf-8")
+        if type(s)==str: s = s.encode("utf-8")
         stream.write(s)
 
 @checks([str])

ocrolib/ligatures.py

@@ -50,15 +50,15 @@ def __init__(self):
         # note that "_" and "~" always have a special meaning
         # but are treated like other ASCII characters
         for i in range(32,1024):
-            self.add(unichr(i),i)
+            self.add(chr(i),i)
         for c in common_chars:
             self.add(c,ord(c))
     def add(self,name,code,override=1):
-        assert type(name)==unicode or not re.search(r'[\x80-\xff]',name)
+        assert type(name)==str or not re.search(r'[\x80-\xff]',name)
         if not override and self.lig2code.get(name) is not None:
             raise Exception("character '%s' (%d) already in ligature table"%(name,self.ord(name)))
         self.lig2code[name] = code
-        self.code2lig[code] = unicode(name)
+        self.code2lig[code] = str(name)
     def ord(self,name):
         if name=="": return 0 # epsilon
         result = self.lig2code.get(name,-1)
@@ -68,7 +68,7 @@ def ord(self,name):
     def chr(self,code):
         result = self.code2lig.get(code,None)
         if code<0: return u"~"
-        if code<0x10000 and result is None: return unichr(code)
+        if code<0x10000 and result is None: return chr(code)
         return result
     def writeText(self,name):
         with open(name,"w") as stream:

ocrolib/lstm.py

@@ -33,10 +33,10 @@
 import matplotlib.pyplot as plt
 from scipy.ndimage import measurements,filters
 
-import common as ocrolib
+from . import common as ocrolib
 from ocrolib.exceptions import RecognitionError
 from ocrolib.edist import levenshtein
-import utils
+from . import utils
 
 initial_range = 0.1
 
@@ -924,7 +924,7 @@ def s2l(self,s):
     def l2s(self,l):
         "Convert a code sequence into a unicode string after recognition."
         l = self.codec.decode(l)
-        return u"".join(l)
+        return "".join(l)
     def trainString(self,xs,s,update=1):
         "Perform training with a string. This uses the codec and normalizer."
         return self.trainSequence(xs,self.s2l(s),update=update)
@@ -957,7 +957,7 @@ def decode(self,l):
         s = [self.code2char.get(c,"~") for c in l]
         return s
 
-ascii_labels = [""," ","~"] + [unichr(x) for x in range(33,126)]
+ascii_labels = [""," ","~"] + [chr(x) for x in range(33,126)]
 
 def ascii_codec():
     "Create a codec containing just ASCII characters."

ocrolib/morph.py

@@ -9,7 +9,7 @@
 from pylab import *
 from scipy.ndimage import morphology,measurements,filters
 from scipy.ndimage.morphology import *
-from toplevel import *
+from .toplevel import *
 
 @checks(ABINARY2)
 def label(image,**kw):
@@ -245,6 +245,9 @@ def renumber_labels(a):
     """Alias for renumber_labels_ordered"""
     return renumber_labels_ordered(a)
 
+def cmp(a, b):
+    return (a > b) - (a < b)
+
 def pyargsort(seq,cmp=cmp,key=lambda x:x):
     """Like numpy's argsort, but using the builtin Python sorting
     function.  Takes an optional cmp."""

ocrolib/psegutils.py

@@ -5,8 +5,8 @@
 import matplotlib.patches as mpatches
 from scipy.ndimage import filters,interpolation
 
-from toplevel import *
-import sl,morph
+from .toplevel import *
+from . import sl,morph
 
 def B(a):
     if a.dtype==np.dtype('B'): return a