diff --git a/OLD/ocrolib/ngraphs.py b/OLD/ocrolib/ngraphs.py index f2fe49a5..7398f9dc 100644 --- a/OLD/ocrolib/ngraphs.py +++ b/OLD/ocrolib/ngraphs.py @@ -88,7 +88,7 @@ def computeNGraphs(self,fnames,n): continue with codecs.open(fname,"r","utf-8") as stream: for lineno,line in enumerate(safe_readlines(stream)): - assert type(line)==unicode + assert isinstance(line, str) if lineno=linelimit+lineskip: break line = line[:-1] diff --git a/ocrolib/__init__.py b/ocrolib/__init__.py index 593a8310..acd64eb2 100644 --- a/ocrolib/__init__.py +++ b/ocrolib/__init__.py @@ -10,6 +10,6 @@ ### top level imports ################################################################ -import default -from common import * -from default import traceback as trace +from . import default +from .common import * +from .default import traceback as trace diff --git a/ocrolib/chars.py b/ocrolib/chars.py index 8ee82c5c..aeb21f4f 100644 --- a/ocrolib/chars.py +++ b/ocrolib/chars.py @@ -4,18 +4,18 @@ # common character sets -digits = u"0123456789" -letters = u"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" -symbols = ur"""!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~""" +digits = "0123456789" +letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" +symbols = r"""!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~""" ascii = digits+letters+symbols -xsymbols = u"""€¢£»«›‹÷©®†‡°∙•◦‣¶§÷¡¿▪▫""" -german = u"ÄäÖöÜüß" -french = u"ÀàÂâÆæÇçÉéÈèÊêËëÎîÏïÔôŒœÙùÛûÜüŸÿ" -turkish = u"ĞğŞşıſ" -greek = u"ΑαΒβΓγΔδΕεΖζΗηΘθΙιΚκΛλΜμΝνΞξΟοΠπΡρΣσςΤτΥυΦφΧχΨψΩω" -portuguese = u"ÁÃÌÍÒÓÕÚáãìíòóõú" -telugu = u" ఁంఃఅఆఇఈఉఊఋఌఎఏఐఒఓఔకఖగఘఙచఛజఝఞటఠడఢణతథదధనపఫబభమయరఱలళవశషసహఽాిీుూృౄెేైొోౌ్ౘౙౠౡౢౣ౦౧౨౩౪౫౬౭౮౯" +xsymbols = """€¢£»«›‹÷©®†‡°∙•◦‣¶§÷¡¿▪▫""" +german = "ÄäÖöÜüß" +french = "ÀàÂâÆæÇçÉéÈèÊêËëÎîÏïÔôŒœÙùÛûÜüŸÿ" +turkish = "ĞğŞşıſ" +greek = "ΑαΒβΓγΔδΕεΖζΗηΘθΙιΚκΛλΜμΝνΞξΟοΠπΡρΣσςΤτΥυΦφΧχΨψΩω" +portuguese = "ÁÃÌÍÒÓÕÚáãìíòóõú" +telugu = " ఁంఃఅఆఇఈఉఊఋఌఎఏఐఒఓఔకఖగఘఙచఛజఝఞటఠడఢణతథదధనపఫబభమయరఱలళవశషసహఽాిీుూృౄెేైొోౌ్ౘౙౠౡౢౣ౦౧౨౩౪౫౬౭౮౯" default = ascii+xsymbols+german+french+portuguese @@ -35,53 +35,53 @@ # there seems to be left vs right leaning, and top-heavy vs bottom-heavy replacements = [ - (u'[_~#]',u"~"), # OCR control characters - (u'"',u"''"), # typewriter double quote - (u"`",u"'"), # grave accent - (u'[“”]',u"''"), # fancy quotes - (u"´",u"'"), # acute accent - (u"[‘’]",u"'"), # left single quotation mark - (u"[“”]",u"''"), # right double quotation mark - (u"“",u"''"), # German quotes - (u"„",u",,"), # German quotes - (u"…",u"..."), # ellipsis - (u"′",u"'"), # prime - (u"″",u"''"), # double prime - (u"‴",u"'''"), # triple prime - (u"〃",u"''"), # ditto mark - (u"µ",u"μ"), # replace micro unit with greek character - (u"[–—]",u"-"), # variant length hyphens - (u"fl",u"fl"), # expand Unicode ligatures - (u"fi",u"fi"), - (u"ff",u"ff"), - (u"ffi",u"ffi"), - (u"ffl",u"ffl"), + ('[_~#]',"~"), # OCR control characters + ('"',"''"), # typewriter double quote + ("`","'"), # grave accent + ('[“”]',"''"), # fancy quotes + ("´","'"), # acute accent + ("[‘’]","'"), # left single quotation mark + ("[“”]","''"), # right double quotation mark + ("“","''"), # German quotes + ("„",",,"), # German quotes + ("…","..."), # ellipsis + ("′","'"), # prime + ("″","''"), # double prime + ("‴","'''"), # triple prime + ("〃","''"), # ditto mark + ("µ","μ"), # replace micro unit with greek character + ("[–—]","-"), # variant length hyphens + ("fl","fl"), # expand Unicode ligatures + ("fi","fi"), + ("ff","ff"), + ("ffi","ffi"), + ("ffl","ffl"), ] def requote(s): - s = unicode(s) - s = re.sub(ur"''",u'"',s) + s = str(s) + s = re.sub(r"''",'"',s) return s def requote_fancy(s,germanic=0): - s = unicode(s) + s = str(s) if germanic: # germanic quoting style reverses the shapes # straight double quotes - s = re.sub(ur"\s+''",u"”",s) - s = re.sub(u"''\s+",u"“",s) - s = re.sub(ur"\s+,,",u"„",s) + s = re.sub(r"\s+''","”",s) + s = re.sub("''\s+","“",s) + s = re.sub(r"\s+,,","„",s) # straight single quotes - s = re.sub(ur"\s+'",u"’",s) - s = re.sub(ur"'\s+",u"‘",s) - s = re.sub(ur"\s+,",u"‚",s) + s = re.sub(r"\s+'","’",s) + s = re.sub(r"'\s+","‘",s) + s = re.sub(r"\s+,","‚",s) else: # straight double quotes - s = re.sub(ur"\s+''",u"“",s) - s = re.sub(ur"''\s+",u"”",s) - s = re.sub(ur"\s+,,",u"„",s) + s = re.sub(r"\s+''","“",s) + s = re.sub(r"''\s+","”",s) + s = re.sub(r"\s+,,","„",s) # straight single quotes - s = re.sub(ur"\s+'",u"‘",s) - s = re.sub(ur"'\s+",u"’",s) - s = re.sub(ur"\s+,",u"‚",s) + s = re.sub(r"\s+'","‘",s) + s = re.sub(r"'\s+","’",s) + s = re.sub(r"\s+,","‚",s) return s diff --git a/ocrolib/common.py b/ocrolib/common.py index 4b5ee87c..881a731e 100644 --- a/ocrolib/common.py +++ b/ocrolib/common.py @@ -13,7 +13,8 @@ import unicodedata import inspect import glob -import cPickle +import pickle as cPickle +import subprocess from ocrolib.exceptions import (BadClassLabel, BadInput, FileNotFound, OcropusException) @@ -25,16 +26,16 @@ from scipy.ndimage import morphology, measurements import PIL -from default import getlocal -from toplevel import (checks, ABINARY2, AINT2, AINT3, BOOL, DARKSEG, GRAYSCALE, +from .default import getlocal +from .toplevel import (checks, ABINARY2, AINT2, AINT3, BOOL, DARKSEG, GRAYSCALE, LIGHTSEG, LINESEG, PAGESEG) -import chars +from . import chars +from . import ligatures +from . import lstm +from . import morph +from . import sl import codecs -import ligatures -import lstm -import morph import multiprocessing -import sl pickle_mode = 2 @@ -47,37 +48,36 @@ def normalize_text(s): """Apply standard Unicode normalizations for OCR. This eliminates common ambiguities and weird unicode characters.""" - s = unicode(s) s = unicodedata.normalize('NFC',s) - s = re.sub(ur'\s+(?u)',' ',s) - s = re.sub(ur'\n(?u)','',s) - s = re.sub(ur'^\s+(?u)','',s) - s = re.sub(ur'\s+$(?u)','',s) + s = re.sub(r'\s+(?u)',' ',s) + s = re.sub(r'\n(?u)','',s) + s = re.sub(r'^\s+(?u)','',s) + s = re.sub(r'\s+$(?u)','',s) for m,r in chars.replacements: - s = re.sub(unicode(m),unicode(r),s) + s = re.sub(m,r,s) return s def project_text(s,kind="exact"): """Project text onto a smaller subset of characters for comparison.""" s = normalize_text(s) - s = re.sub(ur'( *[.] *){4,}',u'....',s) # dot rows - s = re.sub(ur'[~_]',u'',s) # dot rows + s = re.sub(r'( *[.] *){4,}',u'....',s) # dot rows + s = re.sub(r'[~_]',u'',s) # dot rows if kind=="exact": return s if kind=="nospace": - return re.sub(ur'\s','',s) + return re.sub(r'\s','',s) if kind=="spletdig": - return re.sub(ur'[^A-Za-z0-9 ]','',s) + return re.sub(r'[^A-Za-z0-9 ]','',s) if kind=="letdig": - return re.sub(ur'[^A-Za-z0-9]','',s) + return re.sub(r'[^A-Za-z0-9]','',s) if kind=="letters": - return re.sub(ur'[^A-Za-z]','',s) + return re.sub(r'[^A-Za-z]','',s) if kind=="digits": - return re.sub(ur'[^0-9]','',s) + return re.sub(r'[^0-9]','',s) if kind=="lnc": s = s.upper() - return re.sub(ur'[^A-Z]','',s) + return re.sub(r'[^A-Z]','',s) raise BadInput("unknown normalization: "+kind) ################################################################ @@ -420,13 +420,6 @@ def save_object(fname,obj,zip=0): with open(fname,"wb") as stream: cPickle.dump(obj,stream,2) -def unpickle_find_global(mname,cname): - if mname=="lstm.lstm": - return getattr(lstm,cname) - if not mname in sys.modules.keys(): - exec "import "+mname - return getattr(sys.modules[mname],cname) - def load_object(fname,zip=0,nofind=0,verbose=0): """Loads an object from disk. By default, this handles zipped files and searches in the usual places for OCRopus. It also handles some @@ -439,15 +432,12 @@ class names that have changed.""" zip = 1 if zip>0: # with gzip.GzipFile(fname,"rb") as stream: - with os.popen("gunzip < '%s'"%fname,"rb") as stream: - unpickler = cPickle.Unpickler(stream) - unpickler.find_global = unpickle_find_global - return unpickler.load() + gzip = subprocess.Popen(["gzip", "-cd", fname], stdout=subprocess.PIPE) + with gzip.stdout as stream: + return cPickle.load(stream, encoding='latin1') else: with open(fname,"rb") as stream: - unpickler = cPickle.Unpickler(stream) - unpickler.find_global = unpickle_find_global - return unpickler.load() + return pickler.load(stream, encoding='latin1') @@ -504,11 +494,11 @@ def parallel_map(fun,jobs,parallel=0,chunksize=1): def check_valid_class_label(s): """Determines whether the given character is a valid class label. Control characters and spaces are not permitted.""" - if type(s)==unicode: + if type(s)==str: if re.search(r'[\0-\x20]',s): raise BadClassLabel(s) - elif type(s)==str: - if re.search(r'[^\x21-\x7e]',s): + elif type(s)==bytes: + if re.search(rb'[^\x21-\x7e]',s): raise BadClassLabel(s) else: raise BadClassLabel(s) @@ -551,11 +541,11 @@ def allsplitext(path): def base(path): return allsplitext(path)[0] -@checks(str,{str,unicode}) +@checks(str,str) def write_text_simple(file,s): """Write the given string s to the output file.""" with open(file,"w") as stream: - if type(s)==unicode: s = s.encode("utf-8") + if type(s)==str: s = s.encode("utf-8") stream.write(s) @checks([str]) diff --git a/ocrolib/ligatures.py b/ocrolib/ligatures.py index 38e8da0e..09bfaef4 100644 --- a/ocrolib/ligatures.py +++ b/ocrolib/ligatures.py @@ -50,15 +50,15 @@ def __init__(self): # note that "_" and "~" always have a special meaning # but are treated like other ASCII characters for i in range(32,1024): - self.add(unichr(i),i) + self.add(chr(i),i) for c in common_chars: self.add(c,ord(c)) def add(self,name,code,override=1): - assert type(name)==unicode or not re.search(r'[\x80-\xff]',name) + assert type(name)==str or not re.search(r'[\x80-\xff]',name) if not override and self.lig2code.get(name) is not None: raise Exception("character '%s' (%d) already in ligature table"%(name,self.ord(name))) self.lig2code[name] = code - self.code2lig[code] = unicode(name) + self.code2lig[code] = str(name) def ord(self,name): if name=="": return 0 # epsilon result = self.lig2code.get(name,-1) @@ -68,7 +68,7 @@ def ord(self,name): def chr(self,code): result = self.code2lig.get(code,None) if code<0: return u"~" - if code<0x10000 and result is None: return unichr(code) + if code<0x10000 and result is None: return chr(code) return result def writeText(self,name): with open(name,"w") as stream: diff --git a/ocrolib/lstm.py b/ocrolib/lstm.py index f5307590..54c9beaa 100644 --- a/ocrolib/lstm.py +++ b/ocrolib/lstm.py @@ -33,10 +33,10 @@ import matplotlib.pyplot as plt from scipy.ndimage import measurements,filters -import common as ocrolib +from . import common as ocrolib from ocrolib.exceptions import RecognitionError from ocrolib.edist import levenshtein -import utils +from . import utils initial_range = 0.1 @@ -924,7 +924,7 @@ def s2l(self,s): def l2s(self,l): "Convert a code sequence into a unicode string after recognition." l = self.codec.decode(l) - return u"".join(l) + return "".join(l) def trainString(self,xs,s,update=1): "Perform training with a string. This uses the codec and normalizer." return self.trainSequence(xs,self.s2l(s),update=update) @@ -957,7 +957,7 @@ def decode(self,l): s = [self.code2char.get(c,"~") for c in l] return s -ascii_labels = [""," ","~"] + [unichr(x) for x in range(33,126)] +ascii_labels = [""," ","~"] + [chr(x) for x in range(33,126)] def ascii_codec(): "Create a codec containing just ASCII characters." diff --git a/ocrolib/morph.py b/ocrolib/morph.py index c9ff2ba7..4b92b4b6 100644 --- a/ocrolib/morph.py +++ b/ocrolib/morph.py @@ -9,7 +9,7 @@ from pylab import * from scipy.ndimage import morphology,measurements,filters from scipy.ndimage.morphology import * -from toplevel import * +from .toplevel import * @checks(ABINARY2) def label(image,**kw): @@ -245,6 +245,9 @@ def renumber_labels(a): """Alias for renumber_labels_ordered""" return renumber_labels_ordered(a) +def cmp(a, b): + return (a > b) - (a < b) + def pyargsort(seq,cmp=cmp,key=lambda x:x): """Like numpy's argsort, but using the builtin Python sorting function. Takes an optional cmp.""" diff --git a/ocrolib/psegutils.py b/ocrolib/psegutils.py index 63c9bf69..02db53cd 100644 --- a/ocrolib/psegutils.py +++ b/ocrolib/psegutils.py @@ -5,8 +5,8 @@ import matplotlib.patches as mpatches from scipy.ndimage import filters,interpolation -from toplevel import * -import sl,morph +from .toplevel import * +from . import sl,morph def B(a): if a.dtype==np.dtype('B'): return a diff --git a/ocrolib/toplevel.py b/ocrolib/toplevel.py index f393d715..50e9a84f 100644 --- a/ocrolib/toplevel.py +++ b/ocrolib/toplevel.py @@ -5,7 +5,6 @@ import os import sys import warnings -from types import NoneType # FIXME from ... import wrap import numpy as np @@ -172,7 +171,7 @@ def checktype(value,type_): if not np.iterable(value): raise CheckError("expected iterable",value) for x in value: - if not reduce(max,[isinstance(x,t) for t in type_]): + if not any([isinstance(x,t) for t in type_]): raise CheckError("element",x,"of type",type(x),"fails to be of type",type_) return value # for sets, check membership of the type in the set @@ -200,10 +199,10 @@ def argument_check_decorator(f): def argument_checks(*args,**kw): # print("@@@", f, "decl", types, ktypes, "call", # [strc(x) for x in args], kw) - name = f.func_name - argnames = f.func_code.co_varnames[:f.func_code.co_argcount] + name = f.__name__ + argnames = f.__code__.co_varnames[:f.__code__.co_argcount] kw3 = [(var,value,ktypes.get(var,True)) for var,value in kw.items()] - for var,value,type_ in zip(argnames,args,types)+kw3: + for var,value,type_ in list(zip(argnames,args,types))+kw3: try: checktype(value,type_) except AssertionError as e: diff --git a/ocropus-gtedit b/ocropus-gtedit index 4b1999af..f77ac028 100755 --- a/ocropus-gtedit +++ b/ocropus-gtedit @@ -7,7 +7,7 @@ import codecs import re import os.path import base64 -import urllib2 +import urllib from lxml import etree @@ -69,7 +69,7 @@ if args.subparser_name=="write": s = s.strip() if s=="": continue if s[0]=="#": continue - match = re.search(ur'^(\d{4})\.(\d{10})\s*(.*?)$(?i)',s) + match = re.search(r'^(\d{4})\.(\d{10})\s*(.*?)$(?i)',s) if not match: if "[[" not in s: print("???", lineno, ":", s) continue @@ -199,7 +199,7 @@ if args.subparser_name=="html": else: text = u"" with open(fname,"rb") as pngstream: png = pngstream.read() - png = base64.b64encode(png) + png = base64.b64encode(png).decode('ascii') png = "data:image/png;base64,"+png P("") P("",fname) @@ -217,7 +217,7 @@ def url_decode(image): data = image[len(prefix):] data = base64.b64decode(data) else: - data = urllib2.urlopen(image).read() + data = urllib.request.urlopen(image).read() return data if args.subparser_name=="extract": diff --git a/ocropus-hocr b/ocropus-hocr index 43b7222f..a936093a 100755 --- a/ocropus-hocr +++ b/ocropus-hocr @@ -1,6 +1,5 @@ #!/usr/bin/env python -import __builtin__ as python import random as pyrandom import sys import os.path @@ -50,13 +49,13 @@ def PN(*args): E("writing to",args.output) median_xheight = None dirs = [ocrolib.allsplitext(name)[0] for name in args.files] -xhfiles = python.sum([glob.glob(d+"/??????.xheight") for d in dirs],[]) +xhfiles = sum([glob.glob(d+"/??????.xheight") for d in dirs],[]) if len(xhfiles)>5: xheights = [float(ocrolib.read_text(f)) for f in xhfiles] if len(xheights)>0: median_xheight = np.median(xheights) else: - lfiles = python.sum([glob.glob(d+"/??????.bin.png") for d in dirs],[]) + lfiles = sum([glob.glob(d+"/??????.bin.png") for d in dirs],[]) pyrandom.shuffle(lfiles) if len(lfiles)>0: median_xheight = 0.5*np.median([imread(f).shape[0] for f in lfiles[:100]]) diff --git a/ocropus-rpred b/ocropus-rpred index 863947fa..94bc8447 100755 --- a/ocropus-rpred +++ b/ocropus-rpred @@ -272,10 +272,10 @@ def safe_process1(arg): return process1(arg) except IOError as e: if ocrolib.trace: traceback.print_exc() - print_info(fname+":"+e) + print_info(fname+":"+str(e)) except ocrolib.OcropusException as e: if e.trace: traceback.print_exc() - print_info(fname+":"+e) + print_info(fname+":"+sr(e)) except: traceback.print_exc() return None diff --git a/requirements.txt b/requirements.txt index 9712da85..2f5bfd72 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ numpy>=1.9.2 -scipy>=0.15.1 +scipy==1.2.3 matplotlib>=1.4.3 Pillow>=2.7.0 lxml>=3.5.0 diff --git a/run-test b/run-test index 42cb3941..42f6db93 100755 --- a/run-test +++ b/run-test @@ -3,12 +3,12 @@ BASE=$(dirname $0) rm -rf temp -ocropus-nlbin $BASE/tests/testpage.png -o temp -ocropus-gpageseg 'temp/????.bin.png' -ocropus-rpred -n 'temp/????/??????.bin.png' -ocropus-hocr 'temp/????.bin.png' -o temp.html -ocropus-visualize-results temp -ocropus-gtedit html temp/????/??????.bin.png -o temp-correction.html +./ocropus-nlbin $BASE/tests/testpage.png -o temp +./ocropus-gpageseg 'temp/????.bin.png' +./ocropus-rpred -n 'temp/????/??????.bin.png' +./ocropus-hocr 'temp/????.bin.png' -o temp.html +./ocropus-visualize-results temp +./ocropus-gtedit html temp/????/??????.bin.png -o temp-correction.html echo "to see recognition results, type: firefox temp.html" echo "to see correction page, type: firefox temp-correction.html"
%s